from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import argparse
import os.path
import sys

import numpy as np
from six.moves import xrange  # pylint: disable=redefined-builtin
import tensorflow as tf

import input_data
import models
from tensorflow.python.platform import gfile

FLAGS = None

def main(_):
  print(FLAGS.model_size_info)
  # We want to see all the logging messages for this tutorial.
  tf.logging.set_verbosity(tf.logging.INFO)

  # Start a new TensorFlow session.
  sess = tf.InteractiveSession()
  mag_squared_bool = FLAGS.spectrogram_magnitude_squared == 'True'
  # Begin by making sure we have the training data we need. If you already have
  # training data of your own, use `--data_url= ` on the command line to avoid
  # downloading.
  model_settings = models.prepare_model_settings(
      len(input_data.prepare_words_list(FLAGS.wanted_words.split(','))),
      FLAGS.sample_rate, FLAGS.clip_duration_ms, FLAGS.window_size_ms,
      FLAGS.window_stride_ms, FLAGS.dct_coefficient_count,100)
  audio_processor = input_data.AudioProcessor(
      FLAGS.data_url, FLAGS.data_dir, FLAGS.silence_percentage,
      FLAGS.unknown_percentage,
      FLAGS.wanted_words.split(','), FLAGS.validation_percentage,
      FLAGS.testing_percentage, model_settings,FLAGS.input_type, mag_squared_bool)
  fingerprint_size = model_settings['fingerprint_size']
  label_count = model_settings['label_count']
  time_shift_samples = int((FLAGS.time_shift_ms * FLAGS.sample_rate) / 1000)
  # Figure out the learning rates for each training phase. Since it's often
  # effective to have high learning rates at the start of training, followed by
  # lower levels towards the end, the number of steps and learning rates can be
  # specified as comma-separated lists to define the rate at each stage. For
  # example --how_many_training_steps=10000,3000 --learning_rate=0.001,0.0001
  # will run 13,000 training loops in total, with a rate of 0.001 for the first
  # 10,000, and 0.0001 for the final 3,000.
  training_steps_list = list(map(int, FLAGS.how_many_training_steps.split(',')))
  learning_rates_list = list(map(float, FLAGS.learning_rate.split(',')))
  if len(training_steps_list) != len(learning_rates_list):
    raise Exception(
        '--how_many_training_steps and --learning_rate must be equal length '
        'lists, but are %d and %d long instead' % (len(training_steps_list),
                                                   len(learning_rates_list)))

  fingerprint_input = tf.placeholder(
      tf.float32, [None, fingerprint_size], name='fingerprint_input')

  logits, dropout_prob = models.create_model(
      fingerprint_input,
      model_settings,
      FLAGS.model_architecture,
      FLAGS.model_size_info,
      is_training=True)

  # Define loss and optimizer
  ground_truth_input = tf.placeholder(
      tf.int64, [None], name='groundtruth_input')

  # Optionally we can add runtime checks to spot when NaNs or other symptoms of
  # numerical errors start occurring during training.
  control_dependencies = []
  if FLAGS.check_nans:
    checks = tf.add_check_numerics_ops()
    control_dependencies = [checks]

  # Create the back propagation and training evaluation machinery in the graph.
  with tf.name_scope('cross_entropy'):
    cross_entropy_mean = tf.losses.sparse_softmax_cross_entropy(
        labels=ground_truth_input, logits=logits)
  tf.summary.scalar('cross_entropy', cross_entropy_mean)
  with tf.name_scope('train'), tf.control_dependencies(control_dependencies):
    learning_rate_input = tf.placeholder(
        tf.float32, [], name='learning_rate_input')
    train_step = tf.train.GradientDescentOptimizer(
        learning_rate_input).minimize(cross_entropy_mean)
  predicted_indices = tf.argmax(logits, 1)
  correct_prediction = tf.equal(predicted_indices, ground_truth_input)
  confusion_matrix = tf.confusion_matrix(
      ground_truth_input, predicted_indices, num_classes=label_count)
  evaluation_step = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
  tf.summary.scalar('accuracy', evaluation_step)

  global_step = tf.train.get_or_create_global_step()
  increment_global_step = tf.assign(global_step, global_step + 1)

  saver = tf.train.Saver(tf.global_variables())

  # Merge all the summaries and write them out to /tmp/retrain_logs (by default)
  merged_summaries = tf.summary.merge_all()
  train_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/train',
                                       sess.graph)
  validation_writer = tf.summary.FileWriter(FLAGS.summaries_dir + '/validation')

  tf.global_variables_initializer().run()

  start_step = 1

  if FLAGS.start_checkpoint:
    models.load_variables_from_checkpoint(sess, FLAGS.start_checkpoint)
    start_step = global_step.eval(session=sess)

  tf.logging.info('Training from step: %d ', start_step)

  set_size = audio_processor.set_size('testing')
  tf.logging.info('set_size=%d', set_size)
  total_accuracy = 0
  total_conf_matrix = None
  for i in xrange(0, set_size, FLAGS.batch_size):
    test_fingerprints, test_ground_truth = audio_processor.get_data(
        FLAGS.batch_size, i, model_settings,  FLAGS.background_frequency,
        FLAGS.background_volume, 0, 'testing', sess)
    test_accuracy, conf_matrix = sess.run(
        [evaluation_step, confusion_matrix],
        feed_dict={
            fingerprint_input: test_fingerprints,
            ground_truth_input: test_ground_truth,
            dropout_prob: 1.0
        })
    batch_size = min(FLAGS.batch_size, set_size - i)
    total_accuracy += (test_accuracy * batch_size) / set_size
    if total_conf_matrix is None:
      total_conf_matrix = conf_matrix
    else:
      total_conf_matrix += conf_matrix
  tf.logging.info('Confusion Matrix:\n %s' % (total_conf_matrix))
  tf.logging.info('Final test accuracy = %.1f%% (N=%d)' % (total_accuracy * 100,
                                                           set_size))

  text_file = open(FLAGS.result_path+'/accuracy_'+FLAGS.noise_set+".txt", "a")
  text_file.write(str(total_accuracy) + ',')
  text_file.close()

  text_file = open(FLAGS.result_path + '/conf_' + FLAGS.noise_set + "_"+FLAGS.SNR+".txt", "w")
  text_file.write(str(total_conf_matrix))
  text_file.close()

if __name__ == '__main__':
  parser = argparse.ArgumentParser()
  parser.add_argument(
      '--data_url',
      type=str,
      # pylint: disable=line-too-long
      default='',
      # pylint: enable=line-too-long
      help='Location of speech training data archive on the web.')
  parser.add_argument(
      '--data_dir',
      type=str,
      default='/tmp/speech_dataset/',
      help="""\
      Where to download the speech training data to.
      """)
  parser.add_argument(
      '--background_volume',
      type=float,
      default=0.1,
      help="""\
      How loud the background noise should be, between 0 and 1.
      """)
  parser.add_argument(
      '--background_frequency',
      type=float,
      default=0.8,
      help="""\
      How many of the training samples have background noise mixed in.
      """)
  parser.add_argument(
      '--silence_percentage',
      type=float,
      default=10.0,
      help="""\
      How much of the training data should be silence.
      """)
  parser.add_argument(
      '--unknown_percentage',
      type=float,
      default=10.0,
      help="""\
      How much of the training data should be unknown words.
      """)
  parser.add_argument(
      '--time_shift_ms',
      type=float,
      default=100.0,
      help="""\
      Range to randomly shift the training audio by in time.
      """)
  parser.add_argument(
      '--testing_percentage',
      type=int,
      default=10,
      help='What percentage of wavs to use as a test set.')
  parser.add_argument(
      '--validation_percentage',
      type=int,
      default=10,
      help='What percentage of wavs to use as a validation set.')
  parser.add_argument(
      '--sample_rate',
      type=int,
      default=16000,
      help='Expected sample rate of the wavs',)
  parser.add_argument(
      '--clip_duration_ms',
      type=int,
      default=1000,
      help='Expected duration in milliseconds of the wavs',)
  parser.add_argument(
      '--window_size_ms',
      type=float,
      default=30.0,
      help='How long each spectrogram timeslice is',)
  parser.add_argument(
      '--window_stride_ms',
      type=float,
      default=10.0,
      help='How long each spectrogram timeslice is',)
  parser.add_argument(
      '--dct_coefficient_count',
      type=int,
      default=40,
      help='How many bins to use for the MFCC fingerprint',)
  parser.add_argument(
      '--how_many_training_steps',
      type=str,
      default='15000,3000',
      help='How many training loops to run',)
  parser.add_argument(
      '--eval_step_interval',
      type=int,
      default=400,
      help='How often to evaluate the training results.')
  parser.add_argument(
      '--learning_rate',
      type=str,
      default='0.001,0.0001',
      help='How large a learning rate to use when training.')
  parser.add_argument(
      '--batch_size',
      type=int,
      default=100,
      help='How many items to train with at once',)
  parser.add_argument(
      '--summaries_dir',
      type=str,
      default='/tmp/retrain_logs',
      help='Where to save summary logs for TensorBoard.')
  parser.add_argument(
      '--wanted_words',
      type=str,
      default='yes,no,up,down,left,right,on,off,stop,go',
      help='Words to use (others will be added to an unknown label)',)
  parser.add_argument(
      '--train_dir',
      type=str,
      default='/tmp/speech_commands_train',
      help='Directory to write event logs and checkpoint.')
  parser.add_argument(
      '--save_step_interval',
      type=int,
      default=100,
      help='Save model checkpoint every save_steps.')
  parser.add_argument(
      '--start_checkpoint',
      type=str,
      default='',
      help='If specified, restore this pretrained model before any training.')
  parser.add_argument(
      '--model_architecture',
      type=str,
      default='conv',
      help='What model architecture to use')
  parser.add_argument(
      '--check_nans',
      type=bool,
      default=False,
      help='Whether to check for invalid numbers during processing')
  parser.add_argument(
      '--SNR',
      type=str,
      default='',
      help='SNR of test data')
  parser.add_argument(
      '--noise_set',
      type=str,
      default='',
      help='If there are unknown noise tyoes included')
  parser.add_argument(
      '--result_path',
      type=str,
      default='',
      help='path to where results should be put')
  parser.add_argument(
      '--input_type',
      type=str,
      default='MFCC',
      help='MFCC if DCT should be applied, log_mel if not')
  parser.add_argument(
      '--model_size_info',
      type=int,
      nargs="+",
      default=[128,128,128],
      help='Model dimensions - different for various models')
  parser.add_argument(
      '--spectrogram_magnitude_squared',
      type=str,
      default='True',
      help='Whether to use squared magnitude spectrogram or not')

  FLAGS, unparsed = parser.parse_known_args()
  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)


"""
basic
=====
This function is inherited by all devices automatically. It does not need to be explicitly declared, unless you want to change its behaviour

Configurable parameters::

    {
        "label_root" : (optional) the root name of the label property
        "use_label_as_$id" : (optional) - if true then instead of creating a "label" property, the $id property is given the label name  (i.e. human-named $id)
        "always_send_metadata" : ["list", "of", "metadata"] - if defined, then all transmissions will be enriched with these metadata properties
        "no_metadata" : If true then doesn't send metadata
        "labels" : A list of values to use as labels
    }

Device properties created::

    {
        "$id" : a unique random property which looks like a MAC address
        "is_demo_device" : to identify that this is a Synth-created device
        "label" : A human-readable label "Device 0", "Device 1" etc.
    }
"""

import random
import logging
import isodate
from .device import Device
from common import importer

class Basic(Device):
    device_number = 0
    myRandom = random.Random()  # Use our own private random-number generator, so we will repeatably generate the same device ID's regardless of who else is asking for random numbers
    myRandom.seed(1234)
    def __init__(self, instance_name, time, engine, update_callback, context, params):
        self.instance_name = instance_name
        self.creation_time = time
        self.engine = engine
        self.update_callback = update_callback
        if not hasattr(self, "properties"):
            self.properties = {}
        self.model = None   # May get set later if we're in a model
        label_root = "Device "
        use_label_as_id = False
        self.always_send_metadata = None
        self.no_metadata = False
        self.clock_skew = 0 # Clock skew is the amount ADDED to our timestamp when we send, i.e. a positive clock skew means that our clock is fast
        if "basic" in params:   # Will only be there if the basic class has been explictly declared (because user wants to override its behaviour)
            label_root = params["basic"].get("label_root", "Device ")
            use_label_as_id = params["basic"].get("use_label_as_$id", False)
            if "clock_skew_max_advance" in params["basic"]:
                max_advance = isodate.parse_duration(params["basic"].get("clock_skew_max_advance")).total_seconds()
                max_retard = isodate.parse_duration(params["basic"].get("clock_skew_max_retard")).total_seconds()
                skew = Basic.myRandom.random() * (max_advance - max_retard)
                self.clock_skew = max_retard + skew
            self.always_send_metadata = params["basic"].get("always_send_metadata", None)
            self.no_metadata = params["basic"].get("no_metadata", False)

        if not self.no_metadata:
            self.properties["is_demo_device"] = True    # Flag this device so it's easy to delete (only) demo devices from an account that has also started to have real customer devices in it too.

        if "labels" in params.get("basic",{}):
            labs = params["basic"]["labels"]
            label = labs[Basic.device_number % len(labs)]
        else:
            label = label_root + str(Basic.device_number)

        if use_label_as_id:
            self.properties["$id"] = label  # label.replace(" ","_") # Should really replace ALL illegal characters
        else:
            if not "$id" in self.properties:
                self.properties["$id"] = "-".join([format(Basic.myRandom.randrange(0,255),'02x') for i in range(6)])  # A 6-byte MAC address 01-23-45-67-89-ab
            if not self.no_metadata:
                self.properties["label"] = label

        if not self.no_metadata:    # What no_metadata really means is "don't spew stuff out at boot"
            self.do_comms(self.properties, force_comms=True) # Communicate ALL properties on boot (else device and its properties might not be created if comms is down).
        logging.info("Created device " + str(Basic.device_number+1) + " : " + self.properties["$id"])
        Basic.device_number = Basic.device_number + 1
        self.in_property_group = False

    def external_event(self, event_name, arg):
        logging.info("Received external event "+event_name+" for device "+str(self.properties["$id"]))

    def close(self):
        pass

    def comms_ok(self):
        return True

    def set_id(self, new_id):
        """Set device id - call this before super() in child classes if you want to override the id that this base class will normally set """
        if not hasattr(self, "properties"):
            self.properties = {}
        self.properties.update({ "$id" : new_id })

    def enrich_metadata(self, properties):
        for p in self.always_send_metadata:
            if p not in properties:
                if p in self.properties:
                    properties[p] = self.properties[p]
        return properties

    def _transmit(self, the_id, ts, properties):
        if self.always_send_metadata is not None:
            properties = self.enrich_metadata(properties)

        if self.update_callback:
            self.update_callback(the_id, ts, properties)
        else:
            logging.warning("No callback installed to update device properties")

    def transmit(self, the_id, ts, properties, force_comms):
        if not self.comms_ok() and not force_comms:
            return
        # logging.info("Doing transmit")
        self._transmit(the_id, ts, properties)

    # Internal methods

    def do_comms(self, properties, force_comms = False, timestamp = None):
        if timestamp == None:
            timestamp = self.engine.get_now() + self.clock_skew
        if not "$id" in properties: # Ensure there's an ID
            properties["$id"] = self.properties["$id"]
        if not "$ts" in properties: # Ensure there's a timestamp
            properties["$ts"] = timestamp
        # logging.info("About to self.transmit")
        self.transmit(self.properties["$id"], timestamp, properties, force_comms)

    def get_property(self, prop_name, default_value=None):
        assert hasattr(self, "properties"), "basic::get_property('"+prop_name+"') called before basic:__init__() has been called"
        if default_value is not None:
            if self.property_absent(prop_name):
                return default_value
        return self.properties[prop_name]

    def get_property_or_None(self, prop_name):
        if self.property_absent(prop_name):
            return None
        return self.properties[prop_name]

    def property_exists(self, prop_name):
        return prop_name in self.properties

    def property_absent(self, prop_name):
        return not self.property_exists(prop_name)

    def get_properties(self):
        return self.properties

    def set_property(self, prop_name, value,
                     always_send = True,
                     force_send = False,
                     timestamp = None):
        """Set device property and transmit an update"""
        if not prop_name in self.properties:
            changed = True
        elif self.properties[prop_name] != value:
            changed = True
        else:
            changed = False

        if timestamp == None:
            timestamp = self.engine.get_now() + self.clock_skew

        new_props = { prop_name : value, "$id" : self.properties["$id"], "$ts" : timestamp }
        self.properties.update(new_props)
        # logging.info("set_prop")
        if changed or always_send:
            # logging.info("c or as")
            if self.in_property_group:
                # logging.info("in group")
                self.property_group.update(new_props)
            else:
                # logging.info("not in group")
                self.do_comms(new_props, timestamp = timestamp, force_comms = force_send)

    def set_properties(self, new_props):
        np = new_props.copy()
        np.update({ "$id" : self.properties["$id"], "$ts" : self.engine.get_now() + self.clock_skew })  # Force ID and timestamp to be correct
        self.properties.update(np)
        self.do_comms(np)    # TODO: Suppress if unchanged

    def start_property_group(self):
        """Mark the beginning of a group of set_property() updates, which we want to group as a single message"""
        assert self.in_property_group == False
        self.in_property_group = True
        self.property_group = {}

    def end_property_group(self):
        assert self.in_property_group == True
        self.do_comms(self.property_group)

        self.in_property_group = False
        self.property_group = {}


"""
lnPi data classes and routines (:mod:`~lnpy.lnpidata`)
======================================================
"""
################################################################################
# Delayed
from __future__ import annotations

from functools import lru_cache
from typing import TYPE_CHECKING, Iterable

from module_utilities import cached

from ._lazy_imports import np, pd
from .docstrings import docfiller
from .extensions import AccessorMixin
from .utils import labels_to_masks, masks_change_convention

if TYPE_CHECKING:
    from pathlib import Path
    from typing import Any, Mapping, Sequence

    import xarray as xr
    from numpy.typing import ArrayLike, DTypeLike, NDArray
    from typing_extensions import Self

    from . import ensembles
    from ._typing import MaskConvention, MyNDArray

@lru_cache(maxsize=20)
def _get_n_ranges(shape: tuple[int, ...], dtype: DTypeLike) -> list[MyNDArray]:
    return [np.arange(s, dtype=dtype) for s in shape]

@lru_cache(maxsize=20)
def _get_shift(
    shape: tuple[int, ...], dlnz: tuple[float, ...], dtype: DTypeLike
) -> MyNDArray:
    shift = np.zeros([], dtype=dtype)
    for i, (nr, m) in enumerate(zip(_get_n_ranges(shape=shape, dtype=dtype), dlnz)):  # type: ignore
        shift = np.add.outer(shift, nr * m)  # pyright: ignore
    return shift

@lru_cache(maxsize=20)
def _get_data(base: lnPiArray, dlnz: tuple[float, ...]) -> MyNDArray:
    if all(x == 0 for x in dlnz):
        return base.data  # pyright: ignore
    else:
        return (
            _get_shift(base.shape, dlnz, base.data.dtype) + base.data
        )  # pyright: ignore

@lru_cache(maxsize=20)
def _get_maskedarray(
    base: lnPiArray, self: lnPiMasked, dlnz: tuple[float, ...]
) -> np.ma.core.MaskedArray[Any, np.dtype[Any]]:
    return np.ma.MaskedArray(  # type: ignore
        _get_data(base, dlnz),
        mask=self._mask,
        fill_value=base.fill_value,  # pyright: ignore
    )

@lru_cache(maxsize=20)
def _get_filled(
    base: lnPiArray,
    self: lnPiMasked,
    dlnz: tuple[float, ...],
    fill_value: float | None = None,
) -> np.ma.core.MaskedArray[Any, np.dtype[Any]]:
    return _get_maskedarray(base, self, dlnz).filled(fill_value)  # type: ignore

class lnPiArray:
    """
    Wrapper on lnPi lnPiArray

    Parameters
    ----------
    lnz : float or sequence of float
    """

    @docfiller.decorate
    def __init__(
        self,
        lnz: float | ArrayLike,
        data: MyNDArray,
        state_kws: dict[str, Any] | None = None,
        extra_kws: dict[str, Any] | None = None,
        fill_value: float | None = None,
        copy: bool = False,
    ) -> None:
        """
        Parameters
        ----------
        {lnz}
        {data}
        {state_kws}
        {extra_kws}
        {fill_value}
        {copy}
        """

        lnz = np.atleast_1d(lnz)
        data = np.array(data, copy=copy)
        assert data.ndim == len(lnz)

        fill_value = fill_value or np.nan

        if state_kws is None:
            state_kws = {}
        if extra_kws is None:
            extra_kws = {}

        self.data = data
        # make data read only
        self.data.flags.writeable = False

        self.state_kws = state_kws
        self.extra_kws = extra_kws

        self.lnz = lnz
        self.fill_value = fill_value

    @property
    def shape(self) -> tuple[int, ...]:
        return self.data.shape

    def new_like(
        self,
        lnz: float | ArrayLike | None = None,
        data: MyNDArray | None = None,
        copy: bool = False,
    ) -> Self:
        """
        Create new object with optional replacements.

        All parameters are optional.  If not passed, use values in `self`

        Parameters
        ----------
        {lnz}
        {data}
        {copy}

        Returns
        -------
        out : lnPiArray
            New object with optionally updated parameters.

        """
        if lnz is None:
            lnz = self.lnz
        if data is None:
            data = self.data

        return type(self)(
            lnz=lnz,
            data=data,
            copy=copy,
            state_kws=self.state_kws,
            extra_kws=self.extra_kws,
            fill_value=self.fill_value,
        )

    @docfiller.decorate
    def pad(
        self,
        axes: int | Iterable[int] | None = None,
        ffill: bool = True,
        bfill: bool = False,
        limit: int | None = None,
    ) -> Self:
        """
        Pad nan values in underlying data to values

        Parameters
        ----------
        {ffill}
        {bfill}
        {fill_limit}

        Returns
        -------
        out : lnPiArray
            object with padded data
        """

        import bottleneck  # pyright: ignore

        from . import utils

        if axes is None:
            axes = range(self.data.ndim)
        elif isinstance(axes, int):
            axes = (axes,)

        data = self.data
        datas: list[MyNDArray] = []

        if ffill:
            datas += [utils.ffill(data, axis=axis, limit=limit) for axis in axes]
        if bfill:
            datas += [utils.bfill(data, axis=axis, limit=limit) for axis in axes]

        if len(datas) > 0:
            data = bottleneck.nanmean(datas, axis=0)

        new = self.new_like(data=data)
        return new

    def zeromax(self, mask: MyNDArray | bool = False) -> Self:
        """
        Shift values such that lnpi.max() == 0

        Parameters
        ----------
        mask : bool or array-like of bool
            Optional mask to apply to data.  Where `mask` is True,
            data is excluded from calculating maximum.
        """

        data = self.data - np.ma.MaskedArray(self.data, mask).max()  # type: ignore
        return self.new_like(data=data)

@docfiller.decorate
class lnPiMasked(AccessorMixin):
    """
    Masked array like wrapper for lnPi data.

    This is the basic data structure for storing the output from a single TMMC simulation.

    Parameters
    ----------
    {lnz}
    {base}
    {mask_masked}
    {copy}

    Notes
    -----
    Note that in most cases, :class:`lnPiMasked` should not be called directly.
    Rather, a constructor like :meth:`from_data` should be used to construct the object.

    Note the the value of `lnz` is the value to reweight to.

    Basic terminology:

    * T : temperature.
    * k : Boltzmann's constant.
    * beta : Inverse temperature `= 1/(k T)`.
    * mu : chemical potential.
    * lnz : log of activity `= ln(z)`.
    * z : activity `= beta * mu`.
    * lnPi : log of macrostate distribution.

    See Also
    --------
    numpy.ma.MaskedArray
    """

    _DataClass = lnPiArray

    def __init__(
        self,
        lnz: float | ArrayLike,
        base: lnPiArray,
        mask: MyNDArray | None = None,
        copy: bool = False,
    ) -> None:
        lnz = np.atleast_1d(lnz)
        assert lnz.shape == base.lnz.shape

        if mask is None:
            mask = np.full(base.data.shape, fill_value=False, dtype=bool)
        else:
            mask = np.array(mask, copy=copy, dtype=bool)
        assert mask.shape == base.data.shape

        self._mask = mask
        # make mask read-only
        self._mask.flags.writeable = False

        self._base = base
        self._lnz = lnz
        self._dlnz = tuple(self._lnz - self._base.lnz)

        self._cache: dict[str, Any] = {}

    @classmethod
    def from_data(
        cls,
        lnz: float | ArrayLike,
        lnz_data: float | ArrayLike,
        data: MyNDArray,
        mask: MyNDArray | None = None,
        state_kws: dict[str, Any] | None = None,
        extra_kws: dict[str, Any] | None = None,
        fill_value: float | None = None,
        copy: bool = False,
    ) -> Self:
        """
        Create :class:`lnPiMasked` object from raw data.

        Parameters
        ----------
        lnz : float or sequence of float
            Value of `lnz` to reweight data to.
        lnz_data : float or sequence of float
            Value of `lnz` at which `data` was collected
        {data}
        {mask_masked}
        {state_kws}
        {extra_kws}
        {fill_value}
        {copy}

        Returns
        -------
        out : lnPiMasked
        """

        fill_value = fill_value or np.nan

        base = cls._DataClass(
            lnz=lnz_data,
            data=data,
            state_kws=state_kws,
            extra_kws=extra_kws,
            fill_value=fill_value,
            copy=copy,
        )
        return cls(lnz=lnz, base=base, mask=mask, copy=copy)

    @property
    def _data(self) -> MyNDArray:
        return self._base.data  # pyright: ignore

    @property
    def dtype(self) -> np.dtype[Any]:
        """Type (dtype) of underling data"""
        return self._data.dtype

    def _clear_cache(self) -> None:
        self._cache = {}

    @property
    def state_kws(self) -> dict[str, Any]:
        """State variables."""
        return self._base.state_kws

    @property
    def extra_kws(self) -> dict[str, Any]:
        """Extra parameters."""
        return self._base.extra_kws

    @property
    def ma(self) -> np.ma.core.MaskedArray[Any, np.dtype[Any]]:
        """Masked array view of data reweighted data"""
        return _get_maskedarray(self._base, self, self._dlnz)

    def filled(self, fill_value: float | None = None) -> MyNDArray:
        """Filled view or reweighted data"""
        return _get_filled(self._base, self, self._dlnz, fill_value)

    @property
    def data(self) -> MyNDArray:
        """Reweighted data"""
        return _get_data(self._base, self._dlnz)

    @property
    def mask(self) -> MyNDArray:
        """Where `True`, values are masked out."""
        return self._mask

    @property
    def shape(self) -> tuple[int, ...]:
        """Shape of lnPiArray"""
        return self._data.shape

    def __len__(self) -> int:
        return len(self._data)

    @property
    def ndim(self) -> int:
        return self._data.ndim

    @property
    def lnz(self) -> MyNDArray:
        """Value of log(activity) evaluated at"""
        return self._lnz

    @property
    def betamu(self) -> MyNDArray:
        """Alias to `self.lnz`"""
        return self._lnz

    @property
    def volume(self) -> float | None:
        """Accessor to self.state_kws['volume']."""
        return self.state_kws.get("volume", None)

    @property
    def beta(self) -> float | None:
        """Accessor to self.state_kws['beta']."""
        return self.state_kws.get("beta", None)

    def __repr__(self) -> str:
        return f"<lnPi(lnz={self._lnz})>"

    def __str__(self) -> str:
        return repr(self)

    def _index_dict(self, phase: int | str | None = None) -> dict[str, Any]:
        out = {f"lnz_{i}": v for i, v in enumerate(self.lnz)}
        if phase is not None:
            out["phase"] = phase
        # out.update(**self.state_kws)
        return out

    # Parameters for xlnPi
    def _lnpi_tot(self, fill_value: float | None = None) -> MyNDArray:
        return self.filled(fill_value)

    def _pi_params(
        self, fill_value: float | None = None
    ) -> tuple[MyNDArray, float, float]:
        lnpi = self._lnpi_tot(fill_value)

        lnpi_local_max = lnpi.max()
        pi = np.exp(lnpi - lnpi_local_max)
        pi_sum = pi.sum()
        pi_norm = pi / pi_sum

        lnpi_zero = self.data.ravel()[0] - lnpi_local_max

        return pi_norm, pi_sum, lnpi_zero

    @property
    def _lnz_tot(self) -> MyNDArray:
        return self.lnz

    # @cached.meth
    def local_argmax(self, *args: Any, **kwargs: Any) -> tuple[int, ...]:
        """
        Calculate index of maximum of masked data.

        Parameters
        ----------
        *args
            Positional arguments to argmax
        **kwargs
            Keyword arguments to argmax

        See Also
        --------
        numpy.ma.MaskedArray.argmax
        numpy.unravel_index
        """
        return np.unravel_index(self.ma.argmax(*args, **kwargs), self.shape)  # type: ignore

    # @cached.meth
    def local_max(
        self, *args: Any, **kwargs: Any
    ) -> np.ma.core.MaskedArray[Any, np.dtype[Any]]:
        """
        Calculate index of maximum of masked data.

        Parameters
        ----------
        *args
            Positional arguments to argmax
        **kwargs
            Keyword arguments to argmax

        See Also
        --------
        numpy.ma.MaskedArray.max
        """
        return self.ma[self.local_argmax(*args, **kwargs)]  # type: ignore

    # @cached.meth
    def local_maxmask(
        self, *args: Any, **kwargs: Any
    ) -> np.ma.core.MaskedArray[Any, np.dtype[Any]]:
        """Calculate mask where ``self.ma == self.local_max()``"""
        return self.ma == self.local_max(*args, **kwargs)  # type: ignore

    @cached.prop
    def edge_distance_matrix(self) -> NDArray[np.float_]:
        """
        Matrix of distance from each element to a background (i.e., masked) point.

        See Also
        --------
        lnpy.utils.distance_matrix
        """
        from .utils import distance_matrix

        return distance_matrix(~self.mask)

    def edge_distance(self, ref: Self, *args: Any, **kwargs: Any) -> float:
        """
        Distance of local maximum value to nearest background point.

        If `edge_distance` is too small, the value of properties calculated from this
        lnPi cannot be trusted.   This usually is due to the data being reweighted to
        too high a value of `lnz`, or not sampled to sufficiently high values of `N`.

        See Also
        --------
        edge_distance_matrix
        lnpy.utils.distance_matrix
        """
        return ref.edge_distance_matrix[self.local_argmax(*args, **kwargs)]  # type: ignore

    @docfiller.decorate
    def new_like(
        self,
        lnz: float | ArrayLike | None = None,
        base: lnPiArray | None = None,
        mask: MyNDArray | None = None,
        copy: bool = False,
    ) -> Self:
        """
        Create new object with optional parameters

        Parameters
        ----------
        {lnz}
        {base}
        {mask_masked}
        {copy}
        """

        if lnz is None:
            lnz = self._lnz
        if base is None:
            base = self._base
        if mask is None:
            mask = self._mask

        return type(self)(lnz=lnz, base=base, mask=mask, copy=copy)

    @docfiller.decorate
    def pad(
        self,
        axes: int | Iterable[int] | None = None,
        ffill: bool = True,
        bfill: bool = False,
        limit: None = None,
    ) -> Self:
        """
        Pad nan values in underlying data to values

        Parameters
        ----------
        {ffill}
        {bfill}
        {fill_limit}
        inplace : bool, default=False

        Returns
        -------
        out : lnPiMasked
            padded object

        See Also
        --------
        lnPiArray.pad
        """

        base = self._base.pad(axes=axes, ffill=ffill, bfill=bfill, limit=limit)
        return self.new_like(base=base)

    def zeromax(self) -> Self:
        """
        Shift so that lnpi.max() == 0 on reference

        See Also
        --------
        lnPiArray.zeromax
        """

        base = self._base.zeromax(mask=self._mask)
        return self.new_like(base=base)

    def reweight(self, lnz: float | ArrayLike) -> Self:
        """Create new object at specified value of `lnz`"""
        return self.new_like(lnz=lnz)

    def or_mask(self, mask: MyNDArray, **kwargs: Any) -> Self:
        """New object with logical or of self.mask and mask"""
        return self.new_like(mask=(mask | self.mask))

    def and_mask(self, mask: MyNDArray, **kwargs: Any) -> Self:
        """New object with logical and of self.mask and mask"""
        return self.new_like(mask=(mask & self.mask))

    @classmethod
    def from_table(
        cls,
        path: str | Path,
        lnz: float | ArrayLike,
        state_kws: dict[str, Any] | None = None,
        sep: str = r"\s+",
        names: Sequence[str] | None = None,
        csv_kws: Mapping[str, Any] | None = None,
        **kwargs: Any,
    ) -> Self:
        r"""
        Create lnPi object from text file table with columns [n_0,...,n_ndim, lnpi]

        Parameters
        ----------
        path : path-like
            file object to be read
        lnz : array-like
            :math:`\beta \mu` for each component
        state_kws : dict, optional
            define state variables, like volume, beta
        sep : string, optional
            separator for file read
        names : sequence of str
        csv_kws : dict, optional
            optional arguments to `pandas.read_csv`
        **kwargs
            Passed to lnPi constructor
        """
        lnz = np.atleast_1d(lnz)
        ndim = len(lnz)

        if names is None:
            names = [f"n_{i}" for i in range(ndim)] + ["lnpi"]

        if csv_kws is None:
            csv_kws = {}

        da = (
            pd.read_csv(path, sep=sep, names=names, **csv_kws)  # type: ignore
            .set_index(names[:-1])["lnpi"]
            .to_xarray()
        )
        return cls.from_data(
            data=da.values,
            mask=da.isnull().values,
            lnz=lnz,
            lnz_data=lnz,
            state_kws=state_kws,
            **kwargs,
        )

    @classmethod
    def from_dataarray(
        cls, da: xr.DataArray, state_as_attrs: bool | None = None, **kwargs: Any
    ) -> Self:
        """
        Create a lnPi object from :class:`xarray.DataArray`

        Parameters
        ----------
        da : DataArray
            DataArray containing the lnPi data
        state_as_attrs : bool, optional
            If True, get `state_kws` from ``da.attrs``.
        **kwargs
            Extra arguments to :meth:`from_data`

        Returns
        -------
        lnPiMasked

        See Also
        --------
        :meth:`from_data`

        """

        kws: dict[str, Any] = {}
        kws["data"] = da.values
        if "mask" in da.coords:
            kws["mask"] = da.mask.values
        else:
            kws["mask"] = da.isnull().values

        # where are state variables
        if state_as_attrs is None:
            state_as_attrs = bool(da.attrs.get("state_as_attrs", False))
        if state_as_attrs:
            # state variables from attrs
            c = da.attrs
        else:
            c = da.coords  # type: ignore [assignment]

        lnz = []
        state_kws = {}
        for k in da.attrs["dims_state"]:
            val = np.array(c[k])
            if "lnz" in k:
                lnz.append(val)
            else:
                if val.ndim == 0:
                    val = val[()]
                state_kws[k] = val
        kws["lnz"] = lnz
        kws["state_kws"] = state_kws

        kws["lnz_data"] = lnz

        # any overrides
        kwargs = dict(kws, **kwargs)
        return cls.from_data(**kwargs)

    @docfiller.decorate
    def list_from_masks(
        self, masks: Sequence[MyNDArray], convention: MaskConvention = "image"
    ) -> list[Self]:
        """
        Create list of lnpis corresponding to masks[i]

        Parameters
        ----------
        {masks_masked}
        {mask_convention}

        Returns
        -------
        lnpis : list
            list of lnpis corresponding to each mask

        See Also
        --------
        lnpy.utils.masks_change_convention
        """

        return [
            self.or_mask(m) for m in masks_change_convention(masks, convention, False)
        ]

    @docfiller.decorate
    def list_from_labels(
        self,
        labels: MyNDArray,
        features: Sequence[int] | None = None,
        include_boundary: bool = False,
        check_features: bool = True,
        **kwargs: Any,
    ) -> list[Self]:
        """
        Create sequence of lnpis from labels array.

        Parameters
        ----------
        {labels}
        {features}
        {include_boundary}
        {check_features}
        **kwargs
            Extra arguments to to :func:`~lnpy.utils.labels_to_masks`

        Returns
        -------
        outputs : list of lnPiMasked

        See Also
        --------
        lnPiMasked.list_from_masks
        lnpy.utils.labels_to_masks
        """

        masks, features = labels_to_masks(
            labels=labels,
            features=features,
            include_boundary=include_boundary,
            convention=False,
            check_features=check_features,
            **kwargs,
        )
        return self.list_from_masks(masks, convention=False)

    @cached.prop
    @docfiller.decorate
    def xge(self) -> ensembles.xGrandCanonical:
        """{accessor.xge}"""
        from .ensembles import xGrandCanonical

        return xGrandCanonical(self)

    @cached.prop
    @docfiller.decorate
    def xce(self) -> ensembles.xCanonical:
        """{accessor.xce}"""
        from .ensembles import xCanonical

        return xCanonical(self)

# --- Register accessors ---------------------------------------------------------------
# lnPiMasked.register_accessor("xge", xge_accessor)
# lnPiMasked.register_accessor("xce", xce_accessor)

# reveal_type(lnPiMasked.list_from_labels)
# reveal_type(lnPiMasked.xge)


from graphdatascience import GraphDataScience

gds = GraphDataScience("bolt://localhost:7687", auth=("neo4j", "test"))

G, res = gds.graph.project(
    "purchase-proj",
    ["Person", "Product"],
    ["BUYS","CHILD_OF"]
)

G = gds.graph.get("purchase-proj")

n = G.node_count()
props = G.node_properties("Person")


import requests
import urllib.request
from bs4 import BeautifulSoup

site = 'https://weheartit.com'

def get_image(url, file_path, filename):
    full_path = file_path + filename + '.jpeg'
    urllib.request.urlretrieve(url,full_path)

response = requests.get(site)

soup = BeautifulSoup(response.text, 'html.parser')
img_tags = soup.find_all('img')
urls = [img['src'] for img in img_tags]

counter = 0
for url in urls:
    counter = counter + 1
    filename = "{}{}".format('result_image',counter)
    get_image(url, 'result_folder/', filename)


import logging

import pytest

import sdk_install
import sdk_plan
import sdk_marathon

from tests.config import (
    PACKAGE_NAME
)

log = logging.getLogger(__name__)

@pytest.fixture(scope='module', autouse=True)
def configure_package(configure_security):
    try:
        sdk_install.uninstall(PACKAGE_NAME)
        options = {
            "service": {
                "spec_file": "examples/share_pid_namespace.yml"
            }
        }

        sdk_install.install(PACKAGE_NAME, 2, additional_options=options)

        yield # let the test session execute
    finally:
        sdk_install.uninstall(PACKAGE_NAME)

@pytest.mark.sanity
def test_deploy():
    sdk_plan.wait_for_completed_deployment(PACKAGE_NAME)
    deployment_plan = sdk_plan.get_deployment_plan(PACKAGE_NAME)
    log.info("deployment plan: " + str(deployment_plan))

    assert(len(deployment_plan['phases']) == 2)
    assert(deployment_plan['phases'][0]['name'] == 'server-deploy')
    assert(deployment_plan['phases'][1]['name'] == 'once-deploy')
    assert(len(deployment_plan['phases'][0]['steps']) == 2)
    assert(len(deployment_plan['phases'][1]['steps']) == 2)


import argparse
import logging
from os import mkdir
from pathlib import Path
import urllib.request

parser = argparse.ArgumentParser(description='Download a trained SRL model for Dutch')
parser.add_argument(
    '--path',
    dest='path',
    default='models',
    help='Path to the models directory'
)

logger = logging.getLogger(__name__)
logger.addHandler(logging.StreamHandler())

def download_srl_model(datapath='models', name_model=None, name_fasttext=None):
    datapath = Path(datapath)

    if not datapath.exists():
        mkdir(datapath)

    if name_model:
        # explicitly named model
        fname_model = datapath / name_model
    else:
        # default model
        fname_model = datapath / 'srl.pt'

        if not fname_model.exists():
            # download if not found
            logger.info('Downloading srl.pt')
            url = 'https://surfdrive.surf.nl/files/index.php/s/kOgUm0oEpmx5HiZ/download'
            urllib.request.urlretrieve(url, fname_model)
        else:
            logger.info('srl.pt found')

    if name_fasttext:
        # explicitly named fasttext
        fname_fasttext = datapath / name_fasttext
    else:
        # default fasttext
        fname_fasttext = datapath / 'fasttext.model.bin'

        if not fname_fasttext.exists():
            # download if not found
            logger.info('Downloading fasttext.model.bin')
            url = 'https://surfdrive.surf.nl/files/index.php/s/085yxFcRmn0osMw/download'
            urllib.request.urlretrieve(url, fname_fasttext)
        else:
            logger.info('fasttext.model.bin found')

    return str(fname_fasttext), str(fname_model)

if __name__ == '__main__':
    logger.setLevel(logging.DEBUG)

    args = parser.parse_args()
    download_srl_model(args.path)


# -*- coding: utf-8 -*-
# @Time    : 2021/12/2 20:05
# @Email   : 849080458@qq.com
# @Author  : zgh
# @FileName: simple_server_01.py
# @Software: PyCharm

# 搭建服务、监听动作、处理程序、返回数据

from wsgiref.simple_server import make_server

def app(env, start_response):
    # env 获取请求相关数据
    # 状态码，header
    start_response('200 ok', [('content-type', 'text/plain')])
    return [b'hello']

server = make_server('', 9000, app)
server.serve_forever()


import pygame

pygame.init()
HEIGHT, WIDHT = 500, 500
screen = pygame.display.set_mode((HEIGHT, WIDHT))
pygame.display.set_caption("Clock")

runing = True
while runing:
    screen.fill((0, 0, 175))
    for event in pygame.event.get():
        if event.type == pygame.QUIT:
            runing = False
    pygame.draw.line(
        screen,
        color=(255, 255, 255),
        start_pos=(0, 0),
        end_pos=(500, 500),
        width=10
    )
    pygame.draw.line(
        screen,
        color=(255, 255, 255),
        start_pos=(500, 0),
        end_pos=(0, 500),
        width=10
    )
    pygame.draw.circle(
        screen,
        color=(0, 0, 0),
        center=(250, 250),
        radius=80,
        width=10
    )
    pygame.draw.rect(
        screen,
        color=(177, 177, 177),
        rect=pygame.Rect(160, 160, 180, 180),
        width=10
    )
    pygame.draw.polygon(
        screen,
        color=(100, 0, 0),
        points= [(160, 160), (340, 160), (250, 80)],
        width=10
    )
    pygame.display.flip()


from typing import Tuple, List
import sys
import time
import signal
import gc
from Settings import Settings
from Logger import Logger
from FileTools import FileTools
from CrawlerBatchCoordinator import CrawlerBatchCoordinator
from StopCrawlingException import StopCrawlingException
from GlobalVarsHolder import GlobalVarsHolder

class CrawlerMain:
    def __init__(self):
        self._perform_basic_initialization()

        crawled_urls, crawl_queue = self._load_saved_crawled_urls_and_crawl_queue()

        self._continue_running: bool = True
        self._h: GlobalVarsHolder = GlobalVarsHolder(crawled_urls, crawl_queue)

    def start(self) -> None:
        self._start_timeout()
        Logger.log("Starting crawler.", terminal=True)

        self._main_loop()

        self._exit()

    def _perform_basic_initialization(self) -> None:
        Settings.initialize_settings()
        Logger.start_server()
        Settings.create_directories()
        self._set_signal_handlers()

        Logger.log("Basic initialization performed.")

    def _set_signal_handlers(self) -> None:
        for signum in Settings.CAPTURED_SIGNALS:
            signal.signal(signum, self._signal_handler)

            # restart system calls if a signal is received (so the program can finish gracefully without any semi-predictable state changes)
            signal.siginterrupt(signum, False)

    def _signal_handler(self, signum, frame) -> None:
        self._continue_running = False
        Logger.log("A signal requested the program to exit. Please wait for the current batch to finish.", terminal=True)

    def _load_saved_crawled_urls_and_crawl_queue(self) -> Tuple[List[str], List[str]]:
        Logger.log("Loading saved crawled URLs and crawl queue.")

        crawled_urls: List[str] = FileTools(Settings.CRAWLED_URLS_FILE).read_1d_csv_to_list()
        crawl_queue: List[str] = FileTools(Settings.CRAWL_QUEUE_FILE).read_1d_csv_to_list()
        if len(crawl_queue) == 0:  # if the crawl queue is empty, use the start urls
            Logger.log("The crawl queue is empty, using the default start URLs.")
            crawl_queue = list(Settings.START_URLS)

        Logger.log("Crawled URLs ({}) and crawl queue ({}) loaded!".format(len(crawled_urls), len(crawl_queue)))

        return crawled_urls, crawl_queue

    def _start_timeout(self) -> None:
        seconds = Settings.CRAWLER_START_TIMEOUT
        while seconds > 0:
            Logger.log("Crawler starting in {} second(s)...".format(seconds))
            time.sleep(1)
            seconds -= 1

    def _main_loop(self) -> None:
        try:
            while self._continue_running:
                self._inner_main_loop()

        except StopCrawlingException as e:
            Logger.log("The program wants to exit:", str(e), terminal=True)

    def _inner_main_loop(self) -> None:
        coordinator = CrawlerBatchCoordinator(self._h)
        coordinator.coordinator_entry()

        # destroy all unnecessary data before starting another batch
        del coordinator
        gc.collect()

    def _exit(self) -> None:
        Logger.log("Exiting crawler.", terminal=True)
        sys.exit(0)


import glob
import os, utils
from torch.utils.data import DataLoader
from data import datasets, trans
import numpy as np
import torch
from torchvision import transforms
from natsort import natsorted
from model_zoo.transmorph import CONFIGS as CONFIGS_TM
import model_zoo.transmorph as TransMorph
from scipy.ndimage.interpolation import map_coordinates, zoom
from data.data_utils import pkload
from utils.sim_utils import NCC

def main():
    loss_sim = NCC()
    test_dir = '/data/duanbin/image_registration/dataset/OASIS_data/Challenge_test_no_gt/'
    save_dir = '/data/duanbin/image_registration/dataset/OASIS_data/Submit/submission/task_03/'
    model_idx = -2  # -1 is latest_checkpoint
    model_dir = '/data/duanbin/image_registration/experiments/TransMorph-Large-ESC/'
    config = CONFIGS_TM['TransMorph-Large']
    model = TransMorph.TransMorph(config)
    best_model = torch.load(model_dir + natsorted(os.listdir(model_dir))[model_idx])['state_dict']
    print('Best model: {}'.format(natsorted(os.listdir(model_dir))[model_idx]))
    model.load_state_dict(best_model)
    model.cuda()

    file_names = glob.glob(test_dir + '*.pkl')
    with torch.no_grad():
        stdy_idx = 0
        for data in file_names:
            x, y = pkload(data)
            x, y = x[None, None, ...], y[None, None, ...]
            x = np.ascontiguousarray(x)
            y = np.ascontiguousarray(y)
            x, y = torch.from_numpy(x).cuda(), torch.from_numpy(y).cuda()
            file_name = file_names[stdy_idx].split('/')[-1].split('.')[0][2:]
            print(file_name)
            model.eval()
            x_in = torch.cat((x, y), dim=1)
            x_def, flow = model(x_in)
            print(f'similarity: {loss_sim(x_def, y)}')
            flow = flow.cpu().detach().numpy()[0]
            flow = np.array([zoom(flow[i], 0.5, order=2) for i in range(3)]).astype(np.float16)
            print(flow.shape)
            np.savez(save_dir + 'disp_{}.npz'.format(file_name), flow)
            stdy_idx += 1

if __name__ == '__main__':
    '''
    GPU configuration
    '''
    os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID"
    os.environ["CUDA_VISIBLE_DEVICES"] = '6'

    main()


from config import *
import os
from datetime import datetime

def ftp_isdir(path):
    # Проверяет, находится ли по заданному пути папка
    try:
        ftp.cwd(path)
        ftp.cwd('..')
        return True
    except ftplib.error_perm:  # Permanent Error. Code 500 - 599
        return False

def ftp_makedirs(paths, ftp):
    paths = paths.replace('\\', '/').split('/')
    n = len(paths)
    init_path = ftp.pwd()

    for i in range(n):
        ftp.mkd(paths[i])
        ftp.cwd(paths[i])

    ftp.cwd(init_path)

def ftp_upload(path_local, path_server, file_name_time=False, upload_dirs=True):
    '''
    Загружает на сервер по пути path_server локальные файлы, находящиеся по пути path_local.
    file_name_time - если True, то создает папку с уникальным именем в формате "%Дата% %Время%", куда
    будут загружены файлы
    upload_dirs - если True, то помимо файлов из каталога path_local, на сервер будут также загружены
    и какталоги с их содержимым
    '''
    start_time = datetime.now()  # Начало отсчета передачи файлов
    datetime_now = ''
    count_uploaded_files = 0  # Кол-во переданных файлов
    ftp_start_path = ftp.pwd()  # Нужно, чтобы после всех переходов, вернуться в исходное положение

    # Если на сервере не существует указанного пути - создать и перейти в него
    if not ftp_isdir(path_server):
        ftp_makedirs(path_server, ftp)
    ftp.cwd(path_server)

    # Установка текущего времени в качестве имени папки
    if file_name_time:
        datetime_now = str(datetime.now().strftime("%Y-%m-%d %H;%M;%S"))
        ftp.mkd(datetime_now)
        ftp.cwd(datetime_now)

    # Формируем содержимое папки
    local_filenames = os.listdir(path_local)
    for local_filename in local_filenames:
        full_path_file = os.path.join(path_local, local_filename)
        count_uploaded_files += 1

        # Если текущий файл является папкой, то производится рекурсивный вызов закачки
        # файлов из данной локальной папки
        if os.path.isdir(full_path_file):
            if upload_dirs:
                print(f'{count_uploaded_files}) Download directory "{local_filename}":')
                ftp_upload(full_path_file, local_filename)  # Рекурсивный вызов
                print()
            else:
                print(f'{count_uploaded_files}) Directory "{local_filename}" is not a file.')
            continue

        # Закачка файла и вывод информации о нем
        with open(full_path_file, 'rb') as file:
            start_file_time = datetime.now()
            ftp.storbinary('STOR ' + local_filename, file, 1024)

            # Вывод информации о текущем файле в формате: (Время скачивания; Размер файла)
            print(f'{count_uploaded_files}) File "{local_filename}" successfully uploaded.',
                  f'\n(Time: {datetime.now() - start_file_time}; ',
                  f'Size: {os.path.getsize(full_path_file)} bytes)')

    ftp.cwd(ftp_start_path)  # Возврат в исходную папку, откуда функция начала работу

    print()
    print(f'Total download time: {datetime.now() - start_time} hh:mm:ss.')
    print(f'Number of transferred files: {count_uploaded_files}')
    print(f'Uploaded to path: {os.path.join(path_server, str(datetime_now))}')

def ftp_download(path_local='ftp_downloaded', file_name_time=False, download_dirs=True):
    '''
    Скачивает с текущего каталога на сервере содержиме на локальный путь path_local
    file_name_time - аналогичен параметру функции ftp_upload()
    download_dirs - если True, то помимо файлов с сервера будут также скачаны
    и какталоги с их содержимым
    '''
    start_time = datetime.now()
    datetime_now = ''
    count_downloaded_files = 0  # Кол-во переданных файлов

    # Установка текущего времени в качестве имени папки
    if file_name_time:
        datetime_now = str(datetime.now().strftime("%Y-%m-%d %H;%M;%S"))
        path_local = os.path.join(path_local, datetime_now)

    # Если указанной папки не существует - создать
    if not os.path.exists(path_local):
        # os.mkdir()  <- Создает одну папку
        os.makedirs(path_local)  # Создает указанный путь

    # Формируем содержимое папки
    server_filenames = ftp.nlst()
    for server_filename in server_filenames:
        full_path_file = os.path.join(path_local, server_filename)
        count_downloaded_files += 1

        # Если текущий файл является папкой, то производится переход в неё,
        # затем рекурсивный вызов выкачивания файлов из данной папки на сервере,
        # Возврат в исходное состояние
        if ftp_isdir(server_filename):
            if download_dirs:
                print(f'{count_downloaded_files}) Download directory "{server_filename}":')
                ftp.cwd(server_filename)
                ftp_download(full_path_file)  # Рекурсивный вызов
                ftp.cwd('..')
                print()
            else:
                print(f'{count_downloaded_files}) Directory "{server_filename}" is not a file.')
            continue

        # Скачивание файла и вывод информации о нем
        with open(full_path_file, 'wb') as file:
            start_file_time = datetime.now()  # Отсчет от начала скачиванияя текущего файла
            ftp.retrbinary('RETR ' + server_filename, file.write)

            # Вывод информации о текущем файле в формате: (Время скачивания; Размер файла)
            print(f'{count_downloaded_files}) File "{server_filename}" successfully downloaded.',
                  f'\n(Time: {datetime.now() - start_file_time}; ',
                  f'Size: {os.path.getsize(full_path_file)} bytes)')

    print()
    print(f'Total download time: {datetime.now() - start_time} hh:mm:ss.')  # Общее время скачивания
    print(f'Number of transferred files: {count_downloaded_files}')  # Общее число файлов и папок
    print(f'Downloaded to path: {os.path.join(path_local, str(datetime_now))}')


import numpy as np
import torch
import torch.nn as nn
from torch.autograd import Variable
import torch.utils.data as Data
import torch.nn.functional as F
import torchvision
import cv2
import random
import time
import os
import skimage
from skimage import measure

def trainable(net, trainable):
    for para in net.parameters():
        para.requires_grad = trainable

def vgg_init(vgg_loc):
    vgg_model = torchvision.models.vgg16(pretrained = False).cuda()
    vgg_model.load_state_dict(torch.load(vgg_loc))
    trainable(vgg_model, False)

    return vgg_model

class vgg(nn.Module):
    def __init__(self, vgg_model):
        super(vgg, self).__init__()
        self.vgg_layers = vgg_model.features
        self.layer_name_mapping = {
            '1': "relu1_1",
            '3': "relu1_2",
            '6': "relu2_1",
            '8': "relu2_2"
        }

    def forward(self, x):
        output = []
        for name, module in self.vgg_layers._modules.items():
            x = module(x)
            if name in self.layer_name_mapping:
                output.append(x)
        return output

def ensure_exists(dname):
    if not os.path.exists(dname):
        try:
            os.makedirs(dname)
        except:
            pass
    return dname


''' Models '''
from django.contrib.auth.models import User
from django.db import models

class Account(models.Model):
	''' Счет '''
	date = models.DateField('Дата создания', auto_now=True)
	group = models.CharField('Основная группа', max_length=200, blank=True)
	subgroup = models.CharField('Подгруппа', max_length=200, blank=True)
	name = models.CharField('Название', max_length=200, blank=True)
	shortname = models.CharField('Сокращенное название', max_length=50)
	allow_users = models.ManyToManyField(User)

	class Meta:
		''' Метаданные счетов '''
		verbose_name = 'Счет'
		verbose_name_plural = 'Счета'
		ordering = ['shortname']

	def __str__(self):
		return self.shortname

class Transaction(models.Model):
	''' Операции '''
	UNIT_CHOICES = (
		('RUB', 'руб'),
		('KG', 'кг')
	)

	date = models.DateTimeField('Дата и время создания', auto_now_add=True)
	opdate = models.DateField('Дата проведения', null=True)
	debit = models.ForeignKey(
		Account,
		verbose_name='Дебет',
		on_delete=models.CASCADE,
		related_name='+'
	)
	credit = models.ForeignKey(
		Account,
		verbose_name='Кредит',
		on_delete=models.CASCADE,
		related_name='+'
	)
	amount = models.IntegerField('Сумма')
	unit = models.CharField('Ед. изм.', max_length=3, choices=UNIT_CHOICES, default='RUB')
	description = models.CharField('Описание', max_length=200, blank=True)

	class Meta:
		''' Метаданные операций '''
		verbose_name = 'Операция'
		verbose_name_plural = 'Операции'


"""The main application module."""
import os
from asyncio import create_subprocess_shell, subprocess
from pathlib import Path
from shlex import quote

import yaml
from fastapi import FastAPI, HTTPException, Query, status

from deploy_on_onto_ns import __version__
from deploy_on_onto_ns.logger import LOGGER
from deploy_on_onto_ns.models import (
    DEPLOYMENT_SCRIPTS,
    DeployOnOntoNsResponse,
    DeployServices,
)

APP = FastAPI(
    title="Deploy on onto-ns.com",
    version=__version__,
    description=(
        Path(__file__).resolve().parent.parent.resolve() / "README.md"
    ).read_text(encoding="utf8"),
    openapi_prefix="/dlite",
)

@APP.get("/", response_model=DeployOnOntoNsResponse)
async def deploy_service(
    service: str = Query(..., description="The service to deploy"),
    env: list[str] = Query(
        [],
        description="The environment variables to set for the deployment script.",
        regex=r"^[A-Za-z_][A-Za-z0-9_]*=[^=]*$",
    ),
) -> dict[str, str | int]:
    """Deploy `service` on onto-ns.com.

    Run local bash script to deploy `service` on onto-ns.com.

    Parameters:
        service: The service to deploy.
        env: The environment variables to set for the deployment script.

    Returns:
        The response from the deployment service.

    """
    if service not in AVAILABLE_SERVICES:
        LOGGER.error("Service %r does not exist.", service)
        raise HTTPException(
            status_code=status.HTTP_404_NOT_FOUND,
            detail=f"Service {service!r} does not exist.",
        )

    script_env = os.environ.copy()
    for env_var in env:
        key, value = env_var.split("=", 1)
        script_env[key] = value

    script = AVAILABLE_SERVICES[service].as_posix()

    LOGGER.info("Running script %r", script)
    LOGGER.debug("with environment: %r.", script_env)

    process = await create_subprocess_shell(
        f"bash {quote(script)}",
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE,
        env=script_env,
    )
    stdout, stderr = await process.communicate()
    if process.returncode is None:
        LOGGER.error("Process did not return a return code.")
        raise HTTPException(
            status_code=status.HTTP_417_EXPECTATION_FAILED,
            detail="Process did not return a return code.",
        )

    return {
        "service": service,
        "returncode": process.returncode,
        "stdout": stdout.decode("utf8"),
        "stderr": stderr.decode("utf8"),
    }

def available_services() -> dict[str, Path]:
    """Get available services to deploy.

    Returns:
        A mapping from names and aliases of all available services to deploy to the
        path to the deployment script.

    """
    deploy_services_yml = DEPLOYMENT_SCRIPTS / "deploy_services.yml"
    if not deploy_services_yml.exists():
        raise FileNotFoundError(
            "Deployment services file "
            f"{deploy_services_yml.relative_to(Path(__file__).resolve())} does not "
            "exist."
        )

    deploy_services_raw = yaml.safe_load(deploy_services_yml.read_bytes())
    services = DeployServices(**deploy_services_raw)

    mapping: dict[str, Path] = {}
    for service in services.services:
        mapping[service.name] = service.script
        for alias in service.aliases:
            mapping[alias] = service.script

    return mapping

AVAILABLE_SERVICES = available_services()


from collections import namedtuple
import os

from enum import Enum
import colors as c
terminal_size = os.get_terminal_size()
terminal_cols = terminal_size.columns

dividers_basic = {
    "dash": "-" * 50,
    "dot": "." * 50,
    "tilda": "~" * 50,
    "underscore": "_" * 50,
    "*": "*" * 50,
    "equal": "=" * 50,
    "hash": "#" * 50,
    "pipe": "|" * 50,
    "plus": "+" * 50,
    "minus": "-" * 50,
}
dividers_neat = {
    "night": "•☽───────────────────✧˖°˖☆˖°˖✧───────────────────☾•",
    "star": "──────────────────────── ⋆⋅☆⋅⋆ ────────────────────────",
    "cat": "=^..^=   =^..^=   =^..^=    =^..^=    =^..^=    =^..^=    =^..^=",
    "geo": "nunununununununununununununununununununununununununununununun",
    "oOo": ".oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.oOo.",
    "basic": "+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+",
    "smiley": "☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻",
    "line": "──────────────────────────────────────────────────────────────────",
    "twinkle": "✦•〰〰〰〰〰〰〰〰〰〰〰〰〰〰〰〰〰〰•★•〰〰〰〰〰〰〰〰〰〰〰〰〰〰〰〰〰〰•✦",
    "diamond": "--:::------::------------------->◇<--------------------::------:::---",
    "temple": "╬╬════════════════════════════════════════════════════════════════╬╬",
    "wave": "▂ ▃ ▄ ▅ ▆ ▇ █ █ ▇ ▆ ▅ ▄ ▃ ▂",
    "big_wave": "▂▂ ▃▃ ▄▄ ▅▅ ▆▆ ▇▇ ██ ██ ▇▇ ▆▆ ▅▅ ▄▄ ▃▃ ▂▂",
    "skull": "☠◉☠◉☠◉☠◉☠☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻◉☠☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻☹☻",
    "fancy": "▅▄▃▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▂▃▄▅",
    "double": "════════════════════════════════════════════════════════════════════",
    "chain": "⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘⫘",
    "tilda_line": "≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃≃",

}
dividers = {}
dividers.update(dividers_basic)
dividers.update(dividers_neat)

def print_all_dividers():
    for divider in dividers:
        print(dividers[divider])

def print_divider(divider_name="basic"):
    text: str  = dividers.get(divider_name, dividers["basic"])
    print(text.center(terminal_cols))

def get_divider(divider_name="basic"):
    text: str  = dividers.get(divider_name, dividers["basic"])
    return text.center(terminal_cols)

DividerChar = namedtuple("DividerChar", ["name", "char"])
class Divider_Chars(Enum):
    LINE = DividerChar("line", "─")
    DASH = DividerChar("dash", "-")
    DOT = DividerChar("dot", ".")
    DOUBLE = DividerChar("double", "═")
    BOTTOM = DividerChar("bottom", "▁")
    CHAIN = DividerChar("chain", "⫘")
    SQUIGGLE = DividerChar("squiggle", "〰")
    TRIPLE = DividerChar("triple", "☰")
    ABOUT_EQUAL = DividerChar("about-equal", "≈")
    TILDA_LINE = DividerChar("tilda-line", "≃")
    STABLE = DividerChar("stable", "⎍")
    POINT_DOWN = DividerChar("point-down", "☟")
    ARROW_DOWN = DividerChar("arrow-down", "↓")
    NEG_INF = DividerChar("neg-inf", "⧞")
    SCRIBBLE = DividerChar("scrible", "෴")
    TRI_DIAMOND =  DividerChar("tri-diamond", "ᚒ")
    OR = DividerChar("or", "ᚖ")
    MOUNTAIN = DividerChar("mountain", "ᨏ")

class MakeDivider:
    def __init__(self):
        self.divider_chars = Divider_Chars
        self.all_dividers = {e.name: e.value.char for e in self.divider_chars}
    def find_divider(self, name: str) -> DividerChar:
        name = name.upper()
        if name not in self.all_dividers:
            return self.divider_chars.LINE.value
        else:
            return self.divider_chars[name.upper()].value
    def make_divider(self, name: str = "line", length: int = terminal_cols):
        divider = self.find_divider(name)
        return divider.char * length

def test_make_divider():
    make_divider = MakeDivider()
    print(make_divider.make_divider("line"))
    print(make_divider.make_divider("dash"))
    print(make_divider.all_dividers)

divider_maker = MakeDivider()
class DividerCharNames:
    LINE = "line"
    DASH = "dash"
    DOT = "dot"
    DOUBLE = "double"
    CHAIN = "chain"
    BOTTOM = "bottom"
    SQUIGGLE = "squiggle"
    TRIPLE = "triple"
    "☰"

DIVIDER_CHARS = DividerCharNames()
def make_full_divider_special(name: str = "line" ) -> str:
    chars = {
        DIVIDER_CHARS.LINE: "─",
        DIVIDER_CHARS.DASH: "-",
        DIVIDER_CHARS.DOT: ".",
        DIVIDER_CHARS.DOUBLE: "═",
        DIVIDER_CHARS.BOTTOM: "▁",
        DIVIDER_CHARS.CHAIN: "⫘",
        DIVIDER_CHARS.SQUIGGLE: "〰",
        DIVIDER_CHARS.TRIPLE: "☰",

    }
    if name not in chars:
        c = chars["line"]
    else:
        c = chars[name]
    return c * terminal_cols

auto_double_cyan = c.cyan("══"*int(terminal_cols//2))
auto_center_stars = get_divider("star").center(terminal_cols, " ")

basic = get_divider("basic")
double = get_divider("double")


# -*- coding: utf-8 -*-
# @Time    : 2021/12/1 15:14
# @Author  : hujinghua
# @File    : dataDistribute.py
# @Software: IntelliJ IDEA

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from sklearn.impute import SimpleImputer
from sklearn.preprocessing import MinMaxScaler
from sklearn.preprocessing import StandardScaler

def precoess(data, process_way):
    min_max = MinMaxScaler()  # 实例化min-max函数，实现归一化
    z_score = StandardScaler()  # 实例化z-score函数，实现标准化
    if process_way == 'min_max':
        data_1 = pd.DataFrame(min_max.fit_transform(data), columns=data.columns)  #对数据进行min_max归一化
    else :
        data_1 = pd.DataFrame(z_score.fit_transform(data), columns=data.columns)  #对数据进行标准化
    return data_1

# data_1.to_excel(r'C:\Users\Carrot\Desktop\data1.xlsx')  # 将归一化后的文件保存到Excel中
# distribute = data_1.describe()
# distribute.to_excel(r'C:\Users\Carrot\Desktop\data1.xlsx', sheet_name="distribute")  #输出到excel文件中

def plot(data_1, is_x):
    plt.rc("font", family = 'KaiTi')    # 设置plt.show()的字体为楷体
    plt.rcParams['axes.unicode_minus'] = False # 用来显示负号
    if is_x:
        data_1.hist(figsize = (20, 10), bins = 20, sharex = is_x, layout = (3, 5))   # 绘制x轴相同的直方图
    else :
        data_1.hist(figsize = (20, 10), bins = 20, sharex = is_x, layout = (3, 5))   # 绘制大小为(20,20)的直方图
    plt.savefig(r'C:\Users\Carrot\Desktop\picture1.png', dpi = 300, bbox_inches = 'tight')
    plt.show()

if __name__ == '__main__':
    pd.set_option('display.max_columns', None)  # 让describe()的结果全部可见
    data = pd.read_csv(r'C:\Users\Carrot\Desktop\forestfires_data.csv')  #读取数据集
    data_1 = precoess(data, process_way = 'min_max') # min_max为归一化，z_score为标准化
    # plot(data_1, False)
    data_1.to_excel(r'C:\Users\Carrot\Desktop\data1.xlsx')  # 将归一化后的文件保存到Excel中
    # distribute = data_1.describe()
    # distribute.to_excel(r'C:\Users\Carrot\Desktop\data1.xlsx', sheet_name="distribute")  #输出到excel文件中


from django.contrib import admin
from django.urls import include, path
from django.conf.urls import url
from . import views
from django.contrib.staticfiles.urls import staticfiles_urlpatterns

app_name = '[udeskapi]'
urlpatterns = [
    path('', views.home),
    path('cc_force_api/', views.cc_force_api, name='cc_force_api'),
    path('call1/', views.call1, name='call1'),
    path('call2/', views.call2, name='call2'),
    path('call3/', views.call3, name='call3'),
    path('call4/', views.call4, name='call4'),
    path('call5/', views.call5, name='call5'),
    path('call6/', views.call6, name='call6'),
    path('call7/', views.call7, name='call7'),
    path('call8/', views.call8, name='call8'),
    path('call9/', views.call9, name='call9'),
    path('call10/', views.call10, name='call10'),
    path('call11/', views.call11, name='call11'),
    path('call12/', views.call12, name='call12'),
    path('call13/', views.call13, name='call13'),
    path('call14/', views.call14, name='call14'),
    path('call15/', views.call15, name='call15'),
    path('call16/', views.call16, name='call16'),
    path('agent1/', views.agent1, name='agent1'),
    path('agent2/', views.agent2, name='agent2'),
    path('agent3/', views.agent3, name='agent3'),
    path('agent4/', views.agent4, name='agent4'),
    path('agent5/', views.agent5, name='agent5'),
    path('agent6/', views.agent6, name='agent6'),
    path('user_group1/', views.user_group1, name='user_group1'),
    path('user_group2/', views.user_group2, name='user_group2'),
    path('user_group3/', views.user_group3, name='user_group3'),
    path('user_group4/', views.user_group4, name='user_group4'),
    path('user_group5/', views.user_group5, name='user_group5'),
    path('callcenter_analysis1/', views.callcenter_analysis1, name='callcenter_analysis1'),
    path('callcenter_analysis2/', views.callcenter_analysis2, name='callcenter_analysis2'),
    path('callcenter_analysis3/', views.callcenter_analysis3, name='callcenter_analysis3'),
    path('callcenter_analysis4/', views.callcenter_analysis4, name='callcenter_analysis4'),
    path('custom_fields1/', views.custom_fields1, name='custom_fields1'),
    path('custom_fields2/', views.custom_fields2, name='custom_fields2'),
    path('organizations1/', views.organizations1, name='organizations1'),
    path('organizations2/', views.organizations2, name='organizations2'),
    path('organizations3/', views.organizations3, name='organizations3'),
    path('organizations4/', views.organizations4, name='organizations4'),
    path('organizations5/', views.organizations5, name='organizations5'),
    path('organizations6/', views.organizations6, name='organizations6'),
    path('customers3/', views.customers3, name='customers3'),
    path('customers4/', views.customers4, name='customers4'),
    path('customers5/', views.customers5, name='customers5'),
    path('customers6/', views.customers6, name='customers6'),
    path('customers7/', views.customers7, name='customers7'),
    path('customers8/', views.customers8, name='customers8'),
    path('customers9/', views.customers9, name='customers9'),
    path('customers10/', views.customers10, name='customers10'),
    path('customers11/', views.customers11, name='customers11'),
    path('customers12/', views.customers12, name='customers12'),
    path('notes1/', views.notes1, name='notes1'),
    path('notes2/', views.notes2, name='notes2'),
    path('notes3/', views.notes3, name='notes3'),
    path('notes4/', views.notes4, name='notes4'),
    # path('notes5/', views.notes5, name='notes5'),
    # path('notes6/', views.notes6, name='notes6'),

]
urlpatterns += staticfiles_urlpatterns()


#!/usr/bin/python3
'''
    app for registering blueprint and starting flask
'''
from flask import Flask, make_response, jsonify
from flask_cors import CORS
from models import storage
from api.v1.views import app_views
from os import getenv

app = Flask(__name__)
CORS(app, origins="0.0.0.0")
app.register_blueprint(app_views)

@app.teardown_appcontext
def tear_down(self):
    '''
    close query after each session
    '''
    storage.close()

@app.errorhandler(404)
def not_found(error):
    '''
    return JSON formatted 404 status code response
    '''
    return make_response(jsonify({'error': 'Not found'}), 404)

if __name__ == "__main__":
    app.run(host=getenv("HBNB_API_HOST", "0.0.0.0"),
            port=int(getenv("HBNB_API_PORT", "5000")), threaded=True)


from __future__ import annotations

import logging

from typing import TYPE_CHECKING

from airflow.decorators import task
from airflow.operators.email import EmailOperator

from airflow_supporter.service import check_off_dag_op

if TYPE_CHECKING:
    from airflow import DAG

    from airflow_supporter.model import CheckOffDagVariable

logger = logging.getLogger(__name__)

def create_check_off_dag(codv: CheckOffDagVariable, dag: DAG) -> None:
    """
    This will create DAG named 'check_off_dag'

    Returns:
        None
    """

    with dag:
        @task
        def get_off_dag_list_op(exclude_dag_list: list[str]) -> list[str]:
            return check_off_dag_op.get_off_dag_list_func(exclude_dag_list)

        @task
        def turn_on_dag_op(dag_list: list[str]) -> None:
            check_off_dag_op.turn_of_dag_func(dag_list)

        off_dag_list_op = get_off_dag_list_op(codv.exclude_dag_list)

        if codv.email:
            EmailOperator(
                task_id="alert_off_dag_list_op",
                to=codv.email,
                subject="alert check_off_dag",
                html_content=",".join(off_dag_list_op),
            )

        turn_on_dag_op(off_dag_list_op)


import json
from typing import Dict

from core.constants import REDIRECT_URLS
from .amount_models import AmountModel
from .buyer_models import BuyerModel

class PaymentModel:
    token: str
    amount: AmountModel
    buyer: BuyerModel
    redirect_urls: Dict

    def __init__(
        self,
        *,
        token: str,
        buyer: BuyerModel,
        amount: AmountModel,
        urls: Dict = REDIRECT_URLS,
    ):
        self.token = token
        self.amount = amount
        self.buyer = buyer
        self.redirect_urls = urls

    def as_dict(self):
        data = {
            "paymentTokenId": self.token,
            "totalAmount": self.amount.as_dict(),
            "buyer": self.buyer.as_dict(),
            "redirectUrl": self.redirect_urls,
        }

        return data

    def serialize(self):
        return json.dumps(self.as_dict())


# --------------------------------------------------------
# Pytorch Yolov2
# Licensed under The MIT License [see LICENSE for details]
# Written by Jingru Tan
# --------------------------------------------------------

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import torch.nn as nn
import numpy as np
from torch.autograd import Variable
import torch
import torch.nn.functional as F
from util.network import WeightLoader

def conv_bn_leaky(in_channels, out_channels, kernel_size, return_module=False):
    padding = int((kernel_size - 1) / 2)
    layers = [nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size,
                        stride=1, padding=padding, bias=False),
            nn.BatchNorm2d(out_channels),
            nn.LeakyReLU(0.1, inplace=True)]
    # layers = [nn.Conv2d(in_channels, out_channels, kernel_size=kernel_size,
    #                     stride=1, padding=padding, bias=False),
    #           nn.LeakyReLU(0.1, inplace=True)]
    if return_module:
        return nn.Sequential(*layers)
    else:
        return layers

class de_Darknet19(nn.Module):

    cfg = {
        #'layer0': [256, 512, 256, 512, 256, 'M'],
        #'layer1': [128, 256, 128, 'M'],
        # 'layer1': ['M'],
        # 'layer2': [64, 128, 64, 'M'],
        # 'layer3': [32, 'M'],
        # 'layer4': [3]

        'layer2': ['M', 64, 128, 64],
        'layer3': ['M', 32],
        'layer4': ['M', 3]
    }

    def __init__(self):
        super(de_Darknet19, self).__init__()
        self.in_channels = 128

        #self.layer0 = self._make_layers(self.cfg['layer0'])
        #self.layer1 = self._make_layers(self.cfg['layer1'])
        self.layer2 = self._make_layers(self.cfg['layer2'])
        self.layer3 = self._make_layers(self.cfg['layer3'])
        self.layer4 = self._make_layers(self.cfg['layer4'])

    def forward(self, x):
        #x = self.layer0(x)
        #x = self.layer1(x)
        x = self.layer2(x)
        x = self.layer3(x)
        x = self.layer4(x)
        return x

    def _make_layers(self, layer_cfg):
        layers = []

        # set the kernel size of the first conv block = 3
        kernel_size = 3
        for v in layer_cfg:
            if v == 'M':
                #layers += [nn.UpsamplingNearest2d(scale_factor=1)]
                #layers += [nn.Upsample(size=None, scale_factor=2, mode='bilinear', align_corners=True)]
                layers += [nn.ConvTranspose2d(in_channels=self.in_channels, out_channels=self.in_channels,  kernel_size=4, stride=2, padding=1, output_padding=0, bias= False)]
                #layers += [nn.Upsample(size=None, scale_factor=2, mode='bilinear', align_corners=True)]
                #layers += [nn.PixelShuffle(1)]
            elif v == 'S':
                layers += [nn.Upsample(size=None, scale_factor=2, mode='bilinear', align_corners=True)]

            else:
                layers += conv_bn_leaky(self.in_channels, v, kernel_size)
                kernel_size = 1 if kernel_size == 3 else 3
                self.in_channels = v
        return nn.Sequential(*layers)

    # very ugly code !! need to reconstruct
    def load_weights(self, weights_file):
        weights_loader = WeightLoader()
        weights_loader.load(self, weights_file)

#
# if __name__ == '__main__':
#     im = np.random.randn(1, 3, 224, 224)
#     im_variable = Variable(torch.from_numpy(im)).float()
#     model = de_Darknet19()
#     out = model(im_variable)
#     print(out.size())
#     print(model)


"""
@author: jpzxshi
"""
import numpy as np
import matplotlib.pyplot as plt
import learner as ln

def plot(data, net):
    x = data.X_test_np
    f_true = data.y_test_np
    f_pred = net.predict(data.X_test, returnnp=True)

    plt.plot(x, f_true, color='b', label='Ground truth', zorder=0)
    plt.plot(x, f_pred, color='r', label='Prediction', zorder=1)
    plt.scatter(data.X_train_np, data.y_train_np, color='b', label='Learned data', zorder=2)
    plt.title(r'$\sin(x)$', fontsize=20)
    plt.legend()
    plt.savefig('regression.pdf')

def main():
    device = 'cpu' # 'cpu' or 'gpu'
    # FNN
    size = [1, 30, 30, 1]
    activation = 'relu'
    # training
    lr = 0.01
    iterations = 20000
    print_every = 1000

    X_train = (2 * np.pi * np.random.rand(30))[:, None]
    y_train = np.sin(X_train)
    X_test = np.linspace(0, 2 * np.pi, num=100)[:, None]
    y_test = np.sin(X_test)

    data = ln.Data(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test)
    net = ln.nn.FNN(size, activation)
    args = {
        'data': data,
        'net': net,
        'criterion': 'MSE',
        'optimizer': 'adam',
        'lr': lr,
        'iterations': iterations,
        'batch_size': None,
        'print_every': print_every,
        'save': True,
        'callback': None,
        'dtype': 'float',
        'device': device
    }

    ln.Brain.Init(**args)
    ln.Brain.Run()
    ln.Brain.Restore()
    ln.Brain.Output()

    plot(data, ln.Brain.Best_model())

if __name__ == '__main__':
    main()


import random
import pandas as pd
from tqdm import tqdm

source_path = 'data/sbr_data_1M.csv'
df = pd.read_csv(source_path)
user_df = df.groupby('user_id')
columns = df.columns
del df
tmp_list = []
i = 0
for k, v in tqdm(user_df):
    tmp_list.extend(v.values.tolist())
    i += 1
    if i == 500000:
        part = pd.DataFrame(tmp_list, columns=columns)
        part.to_csv('data/task_0.csv', header=True, index=False)
        break

target_path1 = 'data/QK-article.csv'
target_path2 = 'data/QB-video.csv'
target_path3 = 'data/QB-article.csv'

source_df = part['user_id']
source_set = set(source_df.user_id.values.to_list())
target_df1 = pd.read_csv(target_path1)
target_set1 = set(target_df1.user_id.values.to_list())
overlap_set1 = source_set & target_set1
overlap_data1 = target_df1[target_df1.user_id.isin(list(overlap_set1))]

overlap_data1.to_csv('data/task_1.csv', header=True, index=False)

target_df2 = pd.read_csv(target_path2)
target_set2 = set(target_df2.user_id.values.to_list())
overlap_set2 = source_set & target_set2
overlap_data2 = target_df2[target_df2.user_id.isin(list(overlap_set2))]

overlap_data2.to_csv('data/task_2.csv', header=True, index=False)

target_df3 = pd.read_csv(target_path3)
target_set3 = set(target_df3.user_id.values.to_list())
overlap_set3 = source_set & target_set3
overlap_data3 = target_df1[target_df3.user_id.isin(list(overlap_set3))]

overlap_data1.to_csv('data/task_3.csv', header=True, index=False)


# coding: utf-8

import uuid
import datetime
from os import path
from PIL import Image
from StringIO import StringIO

from django.core.files.storage import default_storage

from autowb.db import Model

class ImageFileRepo(Model):
    fields = [
        '_id',
        'name',  # full_name
        'size',  # e.g. 140x140
        'meta',
        'original',
        'versions',
        'created',
    ]

    @classmethod
    def create(cls, **kw):
        data = kw.copy()
        data.update({'created': datetime.datetime.now()})
        model = ImageFile(data)
        model.save()
        return model

    @classmethod
    def get_by_name(cls, name):
        return cls.get({'name': name})

    def url(self):
        return default_storage.url(self.name)

def _random_image_name(modifier=None, ext=None):
    modifier = '_' + modifier if modifier else ''
    if not ext:
        ext = 'jpg'

    return '%s%s.%s' % (uuid.uuid4().hex, modifier, ext)

class ImageDataBase:
    def save_with_random_name(self, path, modifier=None):
        ext = self.format if hasattr(self, 'format') else 'jpg'
        image_path = path + _random_image_name(modifier=modifier, ext=ext)
        return self.save(image_path)

class ImageData(ImageDataBase):
    def __init__(self, rawdata):
        self.rawdata = rawdata
        image = Image.open(StringIO(rawdata.read()))
        self.image = image
        self.width, self.height = image.size
        # if original is empty, this file is the original file itself
        self.original = ''
        self.format = image.format

    def save(self, path):
        self.path = default_storage.save(path, self.rawdata)

        model = ImageFileRepo({
            'name': path,
            'size': self.size_str(),
            'meta': '',
            'original': self.original,
            'versions': [],
            'created': datetime.datetime.now()
        })
        model.save()
        return self.path

    def size_str(self):
        return "%sx%s" % (self.width, self.height)

    @classmethod
    def load_from_path(cls, path):
        image = Image.open(path)
        return cls(image)

class ImageFile:
    def __init__(self, fullname):
        self.fullname = fullname
        self.name, self.ext = path.splitext(fullname)

    def name_of_width(self, width, square=False):
        square = 's' if square else ''
        return '%s_w_%s%s%s' % (self.name, square, width, self.ext)


import json

from django.http import HttpResponse
from django.conf import settings

def render_json(data, code=200):
    """
    自定义render函数, code默认值为200, 返回正常
    """
    result = {
        'data': data,
        'code': code,
    }
    # 开发为了显示清晰使用
    if settings.DEBUG:
        json_str = json.dumps(
            result,
            indent=4,  # 缩进
            # sepatarors=[',', ':'],  # 分隔符
            ensure_ascii=False,  # 编码方式
            sort_keys=True,  # 排序
        )
    else:
        # 实际项目中使用
        json_str = json.dumps(result,
                              # sepatarors=[',', ':'],
                              ensure_ascii=False)
    return HttpResponse(json_str)


import re
import tweepy
import config as cf
from json import loads
from datetime import datetime
import pandas as pd
from os import path
from pymongo import MongoClient, UpdateOne
import config as cf
import emoji
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from string import punctuation

eval_obj = SentimentIntensityAnalyzer()
api_key = cf.twitter_apikey
api_key_secret = cf.twitter_secretapikey
access_token = cf.twitter_accesstoken
access_token_secret = cf.twitter_accesstokensecret
auth = tweepy.OAuthHandler(api_key, api_key_secret)

# set access to user's access key and access secret
auth.set_access_token(access_token, access_token_secret)

# calling the api
api = tweepy.API(auth, wait_on_rate_limit=True)
stats = api.rate_limit_status()

def load_db(df, table_name):
    db = MongoClient(cf.mdb_string)
    mydatabase = db[cf.mdb_db]

    dump_df = df
    updates = []
    for _, row in dump_df.iterrows():
        if table_name == "Trends":
            updates.append(UpdateOne({'Trend_Name': row.get('Trend_Name'), 'WOEID': row.get('WOEID')},
                                     {
                '$set': {
                    'Search_Query': row.get('Search_Query'),
                    'Total_Tweets_Count': row.get('Total_Tweets_Count'),
                    'Pulled_At': row.get('Pulled_At')
                },
                "$currentDate": {"lastModified": True}
            }, upsert=True))
        elif table_name == "Tweets":
            updates.append(UpdateOne({'tweet_id': row.get('tweet_id'), 'WOEID': row.get('WOEID')},
                                     {
                '$set': {
                    'tweet_created_at': row.get('tweet_created_at'),
                    'tweet_raw_text': row.get('tweet_raw_text'),
                    'tweet_cleaned_text': row.get('tweet_cleaned_text'),
                    'tweet_emojis': row.get('tweet_emojis'),
                    'tweet_hashtags': row.get('tweet_hashtags'),
                    'tweet_compound_score': row.get('tweet_compound_score'),
                    'tweet_lang': row.get('tweet_lang'),
                    'tweet_source': row.get('tweet_source'),
                    'tweet_retweet_count': row.get('tweet_retweet_count'),
                    'tweet_like_count': row.get('tweet_like_count'),
                    'tweet_search_query': row.get('tweet_search_query'),
                    'tweet_scrapped_datetime_utc': row.get('tweet_scrapped_datetime_utc')
                },
                "$currentDate": {"lastModified": True}
            }, upsert=True))

    mydatabase[table_name].bulk_write(updates)
    result = mydatabase[table_name].bulk_write(updates)
    # result = result.bulk_api_result
    # print("Summary Inserted: {} Updated: {} Errors: {}".format(
    #     max(result['nInserted'], result['nUpserted']), result['nModified'], max(result['writeErrors'], result['writeConcernErrors'])))
    db.close()

def cleanTweet(text):
    text = re.sub(r'@[A-Za-z0-9_]+', '', text)  # remove @mentions
    text = re.sub(r'\#', '', text)  # remove hash tag
    text = re.sub(r'RT[\s]+', '', text)  # remove RT
    text = re.sub(r'https?:\/\/\S+', '', text)  # remove hyperlink
    text = re.sub(r'\n', ' ', text)  # remove next line character
    text = re.sub(r'[0-9]', '', text)  # Remove numbers
    # remove punctuations
    text = ''.join([char for char in text if char not in punctuation])
    text = text.strip()  # remove whitespaces
    return text

def read_location(filepath):

    if not path.exists(filepath):
        raise FileExistsError("File Doesnot exists!! Please check filepath")
    locations_json = open(filepath, "r")
    locations_json = loads(locations_json.read())
    return locations_json

def pull_trends(woeid, max_trends=50):
    trends_list = []
    trends = api.get_place_trends(woeid)
    for i in range(min(len(trends[0]['trends']), max_trends)):
        # if no tweets counts return by API skip it
        if not trends[0]['trends'][i]['tweet_volume']:
            continue
        trends_list.append(
            [trends[0]['trends'][i]['name'], trends[0]['trends'][i]['query'], trends[0]['trends'][i]['tweet_volume'], woeid, datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")])
    return trends_list

def pull_popular_tweets(topics_list, woeid, max_tweet_topic=20, total_max_tweets=100):

    df_return = pd.DataFrame()

    for topic in topics_list:
        if len(df_return.index) == total_max_tweets:
            break
        tweets = tweepy.Cursor(api.search_tweets, q=topic[1],                         # search for each trending topic
                               # tweets in english , type is "recent"/"popular"
                               lang="en", result_type='popular',
                               tweet_mode='extended', count=50).items(max_tweet_topic)
        tweets_list = list(tweets)
        print("Topic: "+topic[0]+" Pulled_Tweets: "+str(len(tweets_list)))
        for tweet in tweets_list:
            clean_text = cleanTweet(tweet.full_text)
            temp_dict = {
                'tweet_id': tweet.id,
                'WOEID': woeid,
                'tweet_created_at': tweet.created_at.strftime("%Y-%m-%d %H:%M:%S"),
                'tweet_raw_text': tweet.full_text.replace("'", "''"),
                'tweet_cleaned_text': clean_text,
                'tweet_emojis': (lambda x: ', '.join(c for c in x if c in emoji.EMOJI_DATA))(clean_text),
                'tweet_hashtags': (lambda x: ", ".join(re.findall(r"#(\w+)", x)))(tweet.full_text),
                'tweet_compound_score': (lambda x: eval_obj.polarity_scores(x)['compound'])(clean_text),
                'tweet_lang': tweet.lang,
                'tweet_source': tweet.source,
                'tweet_retweet_count': tweet.retweet_count,
                'tweet_like_count': tweet.favorite_count,
                'tweet_search_query': topic[0],
                'tweet_scrapped_datetime_utc': datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
            }
            temp_df = pd.DataFrame([temp_dict])
            df_return = pd.concat([df_return, temp_df], ignore_index=True)
    print("Total Tweets Pulled: "+str(len(df_return.index)))
    return df_return

def build_dataset(filelocation):
    records_trend = 0
    records_tweets = 0
    locations = read_location(filelocation)
    for location in locations:
        final_tweets_df = pd.DataFrame()
        final_trends_df = pd.DataFrame()
        print("\nScraping Tweets for Location: "+location['name'])
        trend_list = pull_trends(location['woeid'], max_trends=20)
        trend_df = pd.DataFrame(trend_list, columns=[
                                'Trend_Name', 'Search_Query', 'Total_Tweets_Count', 'WOEID', 'Pulled_At'])
        trend_df = trend_df.fillna(0)

        final_trends_df = pd.concat(
            [final_trends_df, trend_df], ignore_index=True)

        final_tweets_df = pd.concat([final_tweets_df,
                                     pull_popular_tweets(trend_list, location['woeid'], max_tweet_topic=30, total_max_tweets=10000)], ignore_index=True)
        if len(final_tweets_df.index) == 0:
            continue
        trend_df = pd.merge(trend_df, final_tweets_df.groupby(['WOEID', 'tweet_search_query']).size().reset_index(
            name='counts'), how="left", left_on=['WOEID', 'Trend_Name'], right_on=['WOEID', 'tweet_search_query'])
        trend_df = trend_df.dropna()
        records_trend += len(final_trends_df.index)
        records_tweets += len(final_tweets_df.index)
        load_db(final_tweets_df, "Tweets")
        load_db(final_trends_df, "Trends")

    print("Total Records Pulled\nTweets: {} Trends: {}".format(
        records_tweets, records_trend))
    # return final_tweets_df,final_trends_df

if __name__ == "__main__":
    script_start = datetime.now()
    print("Begining Pulling Data at: " +
          script_start.strftime("%Y-%m-%d %H:%M:%S"))
    build_dataset(filelocation="Locations.json")
    script_end=datetime.now()
    print("Execution Time: {} seconds".format(script_start-script_end))


import json
import os
from dataclasses import dataclass

# STORE_PATH = os.path.join(os.path.dirname(__file__), "store.json")
# Find path to Workflow Data folder
# https://www.alfredapp.com/help/workflows/advanced/variables/
STORE_PATH = os.path.join(os.environ["alfred_workflow_data"], "store.json")

@dataclass
class StoreItem:
    """
    Store item. Includes keyword used to query the URL and the URL itself.
    """

    keyword: str
    url: str

def load_store() -> list[StoreItem]:
    """
    Load store from file.

    Returns:
        list[StoreItem]: Store.
    """
    # Check if directory exists
    if not os.path.exists(os.path.dirname(STORE_PATH)):
        os.makedirs(os.path.dirname(STORE_PATH))

    # Check if store exists
    if not os.path.exists(STORE_PATH):
        return []

    with open(STORE_PATH, "r") as f:
        return [StoreItem(**item) for item in json.load(f)]

def save_store_item(item: StoreItem):
    """
    Save single store to file.

    Args:
        item (StoreItem): Store item to save.
    """
    store = load_store()
    store.append(item)
    save_store(store)

def save_store(items: list[StoreItem]):
    """
    Save store to file.

    Args:
        items (list[StoreItem]): Store items to save.
    """
    if not os.path.exists(os.path.dirname(STORE_PATH)):
        os.makedirs(os.path.dirname(STORE_PATH))

    with open(STORE_PATH, "w") as f:
        json.dump([item.__dict__ for item in items], f)