import os
import types
from collections import abc
from tensorflow.core.protobuf import config_pb2
from tensorflow.core.protobuf import meta_graph_pb2
from tensorflow.python.eager import function
from tensorflow.python.eager import def_function
from tensorflow.python.eager import wrap_function
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework.convert_to_constants import convert_variables_to_constants_v2
from tensorflow.python.framework.errors import InvalidArgumentError
from tensorflow.python.framework.tensor_shape import TensorShape
from tensorflow.python.framework.tensor_spec import TensorSpec
from tensorflow.python.grappler import tf_optimizer
from tensorflow.python.keras.engine.training import Model
from tensorflow.python.platform import tf_logging as logging
from tensorflow.python.util import nest
from tensorflow.compiler.xla.service import hlo_pb2
from tensorflow_neuron.python import meta_graph_util as mgu
from tensorflow_neuron.python import graph_def_util as gdu
from tensorflow_neuron.python import utils
from tensorflow_neuron.python.neuron_cc import list_operators
from tensorflow_neuron.python.hlo.optimize import HloOp
from tensorflow_neuron.python.custom_call import CustomCallLowering
from tensorflow_neuron.python.libtfneuron import libtfneuron
from tensorflow_neuron.python._version import is_tf_v1
from tensorflow_neuron.python.neuron_cc_hlo import _SUPPORTED_OPERATOR_TYPES

def trace(func, example_inputs, subgraph_builder_function=None):
    """
    Description
    -----------

    Trace a ``keras.Model`` or a Python callable that can be decorated by
    ``tf.function``, and return an AWS-Neuron-optimized ``keras.Model`` that
    can execute on AWS Machine Learning Accelerators. Tracing is ideal for
    ``keras.Model`` that accepts a list of ``tf.Tensor`` objects and returns
    a list of ``tf.Tensor`` objects. It is expected that users will provide
    example inputs, and the ``trace`` function will execute ``func``
    symbolically and convert it to a ``keras.Model``.

    The returned ``keras.Model`` will support inference only. Attributes or
    variables held by the original function or ``keras.Model`` will be dropped.

    The returned ``keras.Model`` can be exported as SavedModel and served using
    TensorFlow Serving. Please see :ref:`tensorflow-serving` for more
    information about exporting to saved model and serving using TensorFlow
    Serving.

    Options can be passed to Neuron compiler via the environment variable
    ``NEURON_CC_FLAGS``. For example, the syntax
    ``env NEURON_CC_FLAGS="--neuroncore-pipeline-cores=4"`` directs Neuron
    compiler to compile each subgraph to fit in the specified number of
    NeuronCores. This number can be less than the total available NeuronCores
    on an Inf1 instance. See  :ref:`neuron-compiler-cli-reference` for more
    information about compiler options.

    Arguments
    ---------

    -   **func:** The ``keras.Model`` or function to be traced.
    -   **example_inputs:** A ``tf.Tensor`` or a tuple/list/dict of
        ``tf.Tensor`` objects for tracing the function. When ``example_inputs``
        is a ``tf.Tensor`` or a list of ``tf.Tensor`` objects, we expect
        ``func`` to have calling signature ``func(example_inputs)``. Otherwise,
        the expectation is that inference on ``func`` is done by calling
        ``func(*example_inputs)`` when ``example_inputs`` is a ``tuple``,
        or ``func(**example_inputs)`` when ``example_inputs`` is a ``dict``.
        The case where ``func`` accepts mixed positional and keyword arguments
        is currently unsupported.
    -   **subgraph_builder_function:** (Optional) A callable with signature

        ``subgraph_builder_function(node : NodeDef) -> bool``
        (``NodeDef`` is defined in tensorflow/core/framework/node_def.proto)

        that is used as a call-back function to determine which part of
        the tensorflow GraphDef given by tracing ``func`` will be placed on
        Machine Learning Accelerators.

        If ``subgraph_builder_function`` is not provided, then ``trace`` will
        automatically place operations on Machine Learning Accelerators or
        on CPU to maximize the execution efficiency.

        If it is provided, and ``subgraph_builder_function(node)`` returns
        ``True``, and placing ``node`` on Machine Learning Accelerators
        will not cause deadlocks during execution, then ``trace`` will place
        ``node`` on Machine Learning Accelerators. If
        ``subgraph_builder_function(node)`` returns ``False``, then ``trace``
        will place ``node`` on CPU.

    Returns
    -------

    -  An AWS-Neuron-optimized ``keras.Model``.

    Example Usage
    -------------

    .. code:: python

        import tensorflow as tf
        import tensorflow.neuron as tfn

        input0 = tf.keras.layers.Input(3)
        dense0 = tf.keras.layers.Dense(3)(input0)
        model = tf.keras.Model(inputs=[input0], outputs=[dense0])
        example_inputs = tf.random.uniform([1, 3])
        model_neuron = tfn.trace(model, example_inputs)  # trace

        model_dir = './model_neuron'
        model_neuron.save(model_dir)
        model_neuron_reloaded = tf.keras.models.load_model(model_dir)

    Example Usage with Manual Device Placement Using `subgraph_builder_function`
    -------------

    .. code:: python

        import tensorflow as tf
        import tensorflow.neuron as tfn

        input0 = tf.keras.layers.Input(3)
        dense0 = tf.keras.layers.Dense(3)(input0)
        reshape0 = tf.keras.layers.Reshape([1, 3])(dense0)
        output0 = tf.keras.layers.Dense(2)(reshape0)
        model = tf.keras.Model(inputs=[input0], outputs=[output0])
        example_inputs = tf.random.uniform([1, 3])

        def subgraph_builder_function(node):
            return node.op == 'MatMul'

        model_neuron = tfn.trace(
            model, example_inputs,
            subgraph_builder_function=subgraph_builder_function,
        )

    """
    original_func = func
    func, example_inputs = _func_to_concrete_func(func, example_inputs)
    real_op_count = len(_get_real_ops(func.graph.as_graph_def()))

    dumper = OptionalDumper()
    tfn_args, _ = utils.parse_neuron_cc_flags()

    # Bake constants into graph if extract-weights flag is not set (default)
    if not tfn_args.extract_weights:
        # convert all variables to constants
        with utils.change_grappler_logging_level_according_to_cc_flags():
            try:
                cfunc = convert_variables_to_constants_v2(func, aggressive_inlining=True)
            except InvalidArgumentError as err:
                if all(op.type != 'StatefulPartitionedCall' for op in func.graph.get_operations()):
                    raise err
                error_msg = (
                    "convert_variables_to_constants_v2 failed to unpack a StatefulPartitionedCall"
                    " operator; this may be due to StatefulPartitionedCall wrapping custom"
                    " operators, which is caused by saving or serializing models containing"
                    " custom operators. If you are tracing a keras.Model, try to call tfn.trace"
                    " before saving and reloading the model, and please make sure that model"
                    " layers do not contain StatefulPartitionedCall wrapping custom operators."
                )
                raise InvalidArgumentError(err.node_def, err.op, error_msg)
            except ValueError as err:
                error_msg = (
                    ", to reduce protobuf size you can extract the model's weights from the protobuf"
                    " by passing the '--extract-weights RUNTIME_INSTANCE_TYPE' flag during compilation. "
                    " Example : NEURON_CC_FLAGS='--extract-weights inf1.2xlarge' python your_compile_script.py"
                )
                raise ValueError(str(err) + error_msg)
        raise_for_unhandled_resources(cfunc)
    else:
        if 'inf1' in tfn_args.extract_weights:
            assert utils.using_neuronx() == False, ('It appears you are attempting compile for an'
                                                ' inf1 isntance while using tensorflow_neuronx.'
                                                ' Please either use tensorflow.neuron or compile'
                                                ' for an inf2 or trn1 instance.')
        elif 'trn1' in tfn_args.extract_weights or 'inf2' in tfn_args.extract_weights:
            assert utils.using_neuronx() == True, ('It appears you are attempting compile for an'
                                                ' inf2 or trn1 instance while using tensorflow_neuron.'
                                                ' Please either use tensorflow_neuronx or compile'
                                                ' for an inf1 instance.')
        cfunc = func

    graph_def = cfunc.graph.as_graph_def(add_shapes=True)
    if not any(node.op in {'Placeholder', 'PlaceholderWithDefault'} for node in graph_def.node):
        logging.warning('{} does not seem to have any input; returning an uncompiled callable'.format(func))
        return original_func

    # encode known shapes
    feed_dict = _get_feed_dict(func, example_inputs)
    shape_feed_dict = {name: tensor.shape for name, tensor in feed_dict.items()}
    graph_def = gdu.encode_inferred_shapes(graph_def, shape_feed_dict)

    # call main-graph grappler passes
    graph_def, original_graph_def = _run_shaper_and_fuser(graph_def, feed_dict, func, cfunc, subgraph_builder_function, dumper)
    graph_def = gdu.inline_shape_inputs_in_subgraphs(graph_def)
    dumper.maybe_dump_graph_def_as(graph_def, 'graph_def_fused.pb')
    dumper.maybe_compute_io_tensors(graph_def, original_graph_def, func, feed_dict)

    # call graph_def_util/meta_graph_util passes
    graph_def = gdu.run_graph_def_pass_in_subgraphs(graph_def, gdu.convert_shape_to_constant)
    graph_def = mgu.run_grappler_on_subgraphs(graph_def)
    custom_call_lowering = CustomCallLowering()
    graph_def = gdu.run_graph_def_pass_in_subgraphs(graph_def, custom_call_lowering.lower)
    dumper.maybe_dump_graph_def_as(graph_def, 'graph_def_fused_optimized.pb')
    graph_def = gdu.run_compiler_on_subgraphs(graph_def, dumper)
    graph_def = gdu.run_graph_def_pass_in_subgraphs(graph_def, custom_call_lowering.restore)
    graph_def = gdu.restore_compiler_failures(graph_def, original_graph_def)
    graph_def = gdu.run_graph_def_pass_in_subgraphs(graph_def, gdu.erase_large_constants)
    graph_def = gdu.set_execution_plan(graph_def)
    graph_def = gdu.maybe_relax_placeholder_shapes(graph_def)

    if tfn_args.extract_weights:
        graph_def = gdu.encode_extract_weights_info(graph_def, tfn_args.extract_weights)
        # wrap GraphDef as a WrappedFunction
        cfunc = _wrap_variable_graph_def_as_concrete_function(graph_def, func)
        ordered_weights = _get_ordered_weights(func, original_func)
        # wrap ConcreteFunction as a keras model
        model = AwsNeuronModel(cfunc, func.structured_outputs, real_op_count, ordered_weights=ordered_weights)
    else:
        # wrap GraphDef as a WrappedFunction
        cfunc = _wrap_graph_def_as_concrete_function(graph_def, func)
        # wrap ConcreteFunction as a keras model
        model = AwsNeuronModel(cfunc, func.structured_outputs, real_op_count)

    _make_keras_model_savable(model, example_inputs)

    return model

def _func_to_concrete_func(func, example_inputs):
    if not isinstance(example_inputs, tuple):
        example_inputs = (example_inputs,)
    if not isinstance(func, (def_function.Function, function.ConcreteFunction)):
        input_signature = None
        if all(isinstance(item, ops.Tensor) for item in example_inputs):
            input_signature = [TensorSpec(ts.shape, ts.dtype) for ts in example_inputs]
        func = def_function.function(input_signature=input_signature)(func)
    if not isinstance(func, function.ConcreteFunction):
        func = func.get_concrete_function(*example_inputs)
    return func, example_inputs

def raise_for_unhandled_resources(cfunc):
    resources = [ts for ts in cfunc.inputs if ts.dtype == dtypes.resource]
    if not resources:
        return
    consumers = [ts.consumers() for ts in resources]
    raise TypeError(
        'Found unhandled resource tensors {}, most likely due to the following ops {}. '
        'tfn.trace can only trace pure functions (i. e., functions with all the input data '
        'passed through as function parameters, and all the output data returned through '
        'the function results), but the above unhandled resource tensors would require '
        'an extra initialization step at run-time prior to the actual inference logic '
        'and be passed to the traced function in addition to `example_inputs`. As a result, '
        'we kindly ask you to refactor the computationally expensive part of your model as '
        'a pure function.'.format(resources, consumers)
    )

class OptionalDumper:

    def __init__(self):
        tfn_args, _ = utils.parse_neuron_cc_flags()
        self.dump_prefix = tfn_args.dump_prefix
        self.dump_tensor_map = None

    def enabled(self):
        return self.dump_prefix is not None

    def maybe_dump_graph_def_as(self, graph_def, filename):
        if not self.enabled():
            return
        os.makedirs(self.dump_prefix, exist_ok=True)
        with open(os.path.join(self.dump_prefix, filename), 'wb') as f:
            f.write(graph_def.SerializeToString())

    def maybe_compute_io_tensors(self, graph_def, original_graph_def, func, feed_dict):
        if not self.enabled():
            return
        input_names = _get_input_names(func)
        tensor_name_map = {}
        for nop in gdu.get_neuron_nodes(graph_def):
            for idx, name in enumerate(nop.attr[gdu.knOutputNames].list.s):
                neuron_tensor_name = nop.name if idx == 0 else '{}:{}'.format(nop.name, idx)
                tensor_name_map[neuron_tensor_name] = name.decode()
        self.dump_tensor_map = {}
        dump_tensor_names = []
        for nop in gdu.get_neuron_nodes(graph_def):
            in_names = [tensor_name_map.get(name, name) for name in nop.input if not name.startswith('^')]
            in_names = [name if ':' in name else '{}:0'.format(name) for name in in_names]
            out_names = [name.decode() for name in nop.attr[gdu.knOutputNames].list.s]
            dump_tensor_names.extend(in_names)
            dump_tensor_names.extend(out_names)
            self.dump_tensor_map[nop.name] = in_names, out_names
        dumper_cfunc = wrap_function.function_from_graph_def(original_graph_def, input_names, dump_tensor_names)
        flat_inputs = [feed_dict[name] for name in input_names]
        dumper_outputs = dumper_cfunc(*flat_inputs)
        name_to_tensor = {name: tensor for name, tensor in zip(dump_tensor_names, dumper_outputs)}
        for op_name, (in_names, out_names) in self.dump_tensor_map.items():
            in_tensors = [name_to_tensor[name] for name in in_names]
            out_tensors = [name_to_tensor[name] for name in out_names]
            self.dump_tensor_map[op_name] = in_tensors, out_tensors

    def maybe_dump_hlo_snapshots_with_inputs_outputs(self, hlo_opt):
        if not self.enabled():
            return
        for op_name, (inputs, outputs) in self.dump_tensor_map.items():
            inputs = [tensor.numpy() for tensor in inputs]
            outputs = [tensor.numpy() for tensor in outputs]
            workdir = os.path.join(self.dump_prefix, op_name)
            if not os.path.isdir(workdir):
                continue
            hlo_ss_path = os.path.join(workdir, 'hlo_snapshot.pb')
            if os.path.isfile(hlo_ss_path):
                hlo_ss = hlo_pb2.HloSnapshot()
                with open(hlo_ss_path, 'rb') as f:
                    hlo_ss.ParseFromString(f.read())
                _embed_inputs_outputs_into_hlo_snapshot(hlo_ss, inputs, outputs)
                with open(hlo_ss_path, 'wb') as f:
                    f.write(hlo_ss.SerializeToString())
            hlo_ss_opt = hlo_opt.get_snapshot()
            if hlo_opt.input_shuffles is not None:
                inputs = [_shuffle(val, shf) for val, shf in zip(inputs, hlo_opt.input_shuffles)]
            _embed_inputs_outputs_into_hlo_snapshot(hlo_ss_opt, inputs, outputs)
            hlo_ss_opt_path = os.path.join(workdir, 'hlo_snapshot_opt.pb')
            with open(hlo_ss_opt_path, 'wb') as f:
                f.write(hlo_ss_opt.SerializeToString())

def _embed_inputs_outputs_into_hlo_snapshot(hlo_snapshot, inputs, outputs):
    hps = hlo_snapshot.hlo.hlo_module.host_program_shape
    iter_inputs = zip(hps.parameters, inputs), hlo_snapshot.arguments
    iter_outputs = zip(hps.result.tuple_shapes, outputs), hlo_snapshot.result.tuple_literals
    for iterator, args in iter_inputs, iter_outputs:
        for arg_shape, value in iterator:
            arg = args.add()
            arg.shape.CopyFrom(arg_shape)
            attr_name = HloOp.xla_dtype_to_literal_attr_name[arg.shape.element_type]
            literals = getattr(arg, attr_name)
            if isinstance(literals, bytes):
                setattr(arg, attr_name, value.tobytes())
            else:
                literals[:] = value.ravel()

def _shuffle(value, shuffle):
    return value.ravel()[shuffle].reshape(value.shape)

class AwsNeuronModel(Model):

    def __init__(self, func, structured_outputs, real_op_count=None, ordered_weights=None):
        super().__init__(trainable=False, autocast=False)
        self.aws_neuron_function = func
        self._aws_neuron_output_type = None
        self._ordered_weights = ordered_weights
        # this class is sometimes inherited namely from the auto multicore
        # so we cannot assume that there will always be a real_op_count
        if real_op_count is not None:
            self.on_neuron_ratio = self.calculate_on_neuron_ratio(real_op_count)
            if self.on_neuron_ratio < .5:
                logging.warning(f'Warning: Your traced model has {self.on_neuron_ratio * 100}% of operators compiled to neuron.')
        if isinstance(structured_outputs, abc.Mapping):
            self._aws_neuron_output_type = type(structured_outputs)

    def call(self, inputs, *args):
        flat_inputs = nest.flatten((inputs, args))
        if isinstance(inputs, abc.Mapping) and not args:
            if set(inputs.keys()) == set(self.aws_neuron_function._arg_keywords):
                flat_inputs = [inputs[kw] for kw in self.aws_neuron_function._arg_keywords]
        if self._ordered_weights is None:
            outputs = self.aws_neuron_function(*flat_inputs)
        else:
            outputs = self.aws_neuron_function(*flat_inputs, *self._ordered_weights)
        if self._aws_neuron_output_type is not None:
            outputs = self._aws_neuron_output_type(**outputs)
        return outputs

    # Calculates the ratio between the number of original ops in the graph
    # and the number ops compiled to neuron.
    # It ignores Placeholder, Idnetity, ReadVariableOp, NoOp, and NeuronOp
    # The first four being very insignifcant ops and the last representing
    # all the successfully compiled ops.
    def calculate_on_neuron_ratio(self, real_op_count):
        non_compiled_op_count = 0
        found_at_least_one_neuron_op = False
        for op in self.aws_neuron_function.graph.get_operations():
            if op.type not in {'Placeholder', 'IdentityN', 'NeuronOp', 'ReadVariableOp', 'NoOp'}:
                non_compiled_op_count += 1
            elif op.type == 'NeuronOp':
                found_at_least_one_neuron_op = True

        if found_at_least_one_neuron_op:
            return 1 - (non_compiled_op_count/real_op_count)
        return 0

def _get_real_ops(graph_def):
    real_ops = []
    for node in graph_def.node:
        if node.op not in {'Placeholder', 'ReadVariableOp', 'Identity', 'NoOp'}:
            real_ops.append(node.op)

    return real_ops

def _get_feed_dict(func, inputs):
    if len(inputs) == 1:
        inputs, = inputs
    input_names = _get_input_names(func)
    inputs_list = [inputs] if isinstance(inputs, ops.Tensor) else inputs
    inputs_list = nest.flatten(inputs_list)
    return {name: ts for name, ts in zip(input_names, inputs_list)}

def _run_shaper_and_fuser(graph_def, feed_dict, func, cfunc, subgraph_builder_function, dumper):
    new_graph_def, original_graph_def = _run_grappler_on_main_graph(graph_def, cfunc, subgraph_builder_function, dumper)
    name_mapping = {}
    for node in gdu.get_neuron_nodes(new_graph_def):
        output_names = node.attr[gdu.knOutputNames].list.s
        for port, name in enumerate(output_names):
            name_mapping['{}:{}'.format(node.name, port)] = name.decode()
    need_shape_names = []
    for node in gdu.get_neuron_nodes(new_graph_def):
        is_compilable, _ = gdu.neuron_node_is_compilable(node)
        if not is_compilable:
            input_shapes = node.attr[gdu.knInputShapes].list.shape
            for name, shape in zip(node.input, input_shapes):
                if not TensorShape(shape).is_fully_defined():
                    if ':' not in name:
                        name = '{}:0'.format(name)
                    need_shape_names.append(name_mapping.get(name, name))
    if need_shape_names:
        logging.warning('determining {} tensor shapes concretely from runtime'.format(len(need_shape_names)))
        input_names = _get_input_names(func)
        shaper_cfunc = wrap_function.function_from_graph_def(graph_def, input_names, need_shape_names)
        flat_inputs = [feed_dict[name] for name in input_names]
        shaper_outputs = shaper_cfunc(*flat_inputs)
        shape_feed_dict = {name: tensor.shape for name, tensor in feed_dict.items()}
        shape_feed_dict.update((name, ts.shape) for name, ts in zip(need_shape_names, shaper_outputs))
        graph_def = gdu.encode_inferred_shapes(graph_def, shape_feed_dict)
        graph_def, original_graph_def = _run_grappler_on_main_graph(graph_def, cfunc, subgraph_builder_function, dumper)
    else:
        graph_def = new_graph_def
    return graph_def, original_graph_def

def _run_grappler_on_main_graph(graph_def, cfunc, subgraph_builder_function, dumper):
    if not libtfneuron.available:
        return _run_grappler_on_main_graph_legacy(graph_def, cfunc, subgraph_builder_function, dumper)
    opt_config, meta_graph_def = _build_optimize_graph_args(graph_def, cfunc)
    signature_def = meta_graph_def.signature_def['serving_default']
    rewriter_config = opt_config.graph_options.rewrite_options
    optimizers = rewriter_config.optimizers
    optimizers.append('debug_stripper')
    pruning_passes = ['pruning', 'dependency']
    optimizers.extend(pruning_passes)

    with utils.change_grappler_logging_level_according_to_cc_flags():
        graph_def = tf_optimizer.OptimizeGraph(opt_config, meta_graph_def)
    graph_def = original_graph_def = _neuron_optimize_graph_def(graph_def)
    dumper.maybe_dump_graph_def_as(graph_def, 'graph_def_shaped.pb')

    # configure and run operator fusion
    mgu.setup_opt_config_node(graph_def, signature_def, list_operators(), subgraph_builder_function)
    graph_def = _neuron_convert_graph_def(graph_def)
    if subgraph_builder_function is not None:
        opt_config, meta_graph_def = _build_optimize_graph_args(graph_def, cfunc)
        optimizers = opt_config.graph_options.rewrite_options.optimizers
        optimizers.extend(pruning_passes)
        with utils.change_grappler_logging_level_according_to_cc_flags():
            graph_def = tf_optimizer.OptimizeGraph(opt_config, meta_graph_def)
    return graph_def, original_graph_def

def _neuron_optimize_graph_def(graph_def):
    s_graph_def = libtfneuron.NeuronOptimize(graph_def.SerializeToString())
    graph_def.Clear()
    graph_def.ParseFromString(s_graph_def)
    return graph_def

def _neuron_convert_graph_def(graph_def):
    s_graph_def = libtfneuron.NeuronConvert(graph_def.SerializeToString())
    graph_def.Clear()
    graph_def.ParseFromString(s_graph_def)
    return graph_def

def _run_grappler_on_main_graph_legacy(graph_def, cfunc, subgraph_builder_function, dumper):
    # produce GraphDef by running grappler passes
    fuser_automatic = subgraph_builder_function is None
    original_graph_def = graph_def
    opt_config, meta_graph_def = _build_optimize_graph_args(graph_def, cfunc)
    signature_def = meta_graph_def.signature_def['serving_default']
    rewriter_config = opt_config.graph_options.rewrite_options
    optimizers = rewriter_config.optimizers
    optimizers.append('debug_stripper')
    pruning_passes = ['pruning', 'dependency']
    optimizers.extend(pruning_passes)
    optimizers.append('aws_neuron_static_shape_inference')
    optimizers.append('aws_neuron_split_conv2d_same_padding')
    optimizers.append('aws_neuron_static_shape_inference')
    optimizers.append('aws_neuron_mark_ops_in_fixed_shape_context')

    if dumper.enabled() or not fuser_automatic:
        # if dumper is enabled or subgraph_builder_function is given, then run pre-fusing passes
        # and the fuser pass in separate sessions so that Python code can run in the middle
        with utils.change_grappler_logging_level_according_to_cc_flags():
            graph_def = original_graph_def = tf_optimizer.OptimizeGraph(opt_config, meta_graph_def)
        dumper.maybe_dump_graph_def_as(graph_def, 'graph_def_shaped.pb')
        opt_config, meta_graph_def = _build_optimize_graph_args(graph_def, cfunc)
        rewriter_config = opt_config.graph_options.rewrite_options
        optimizers = rewriter_config.optimizers
        optimizers.append('debug_stripper')

    # configure operator fusion
    optimizers.append('aws_neuron_fuse_supported_operators')
    mgu.setup_opt_config_node(meta_graph_def.graph_def, signature_def, list_operators(), subgraph_builder_function)

    # call all grappler passes
    with utils.change_grappler_logging_level_according_to_cc_flags():
        graph_def = tf_optimizer.OptimizeGraph(opt_config, meta_graph_def)
    if not fuser_automatic:
        opt_config, meta_graph_def = _build_optimize_graph_args(graph_def, cfunc)
        optimizers = opt_config.graph_options.rewrite_options.optimizers
        optimizers.extend(pruning_passes)
        with utils.change_grappler_logging_level_according_to_cc_flags():
            graph_def = tf_optimizer.OptimizeGraph(opt_config, meta_graph_def)
    return graph_def, original_graph_def

def _build_optimize_graph_args(graph_def, cfunc):
    meta_graph_def = meta_graph_pb2.MetaGraphDef(graph_def=graph_def)
    sig_def = mgu.build_signature_def(cfunc.inputs, cfunc.outputs)
    meta_graph_def.signature_def['serving_default'].CopyFrom(sig_def)
    opt_config = config_pb2.ConfigProto()
    rewriter_config = opt_config.graph_options.rewrite_options
    rewriter_config.meta_optimizer_iterations = 1
    rewriter_config.min_graph_nodes = -1
    return opt_config, meta_graph_def

def _get_input_names(func):
    captured_inputs = {ts.name for _, ts in func.graph.captures}
    return [ts.name for ts in func.inputs if ts.name not in captured_inputs]

def _get_all_names(func):
    return [ts.name for ts in func.inputs]

def _get_output_names(func):
    outputs, structured_outputs = func.outputs, func.structured_outputs
    if isinstance(structured_outputs, dict):
        # return a map from output argument name to symbolic tensor name
        # in order to let the WrappedFunction's return dictionary have the correct keys
        tensor_specs = nest.flatten(structured_outputs, expand_composites=True)
        tensor_spec_name_map = {spec.name: name for name, spec in structured_outputs.items()}
        tensor_spec_names = [tensor_spec_name_map[spec.name] for spec in tensor_specs]
        return {name: ts.name for ts, name in zip(outputs, tensor_spec_names)}
    elif len(outputs) == 1 and isinstance(structured_outputs, ops.Tensor):
        output, = outputs
        return output.name
    else:
        return [ts.name for ts in outputs]

def _get_ordered_weights(func, original_func):
    # figure out proper order of weights
    # python functions passed into trace have their
    # "weights" stored in captured_inputs as they
    # don't really have a graph

    #function only used when --reduce-neff-size flag is passed
    if isinstance(original_func, types.FunctionType):
        ordered_weights = func.captured_inputs
    else:
        ref_to_var = {var.handle.ref(): var for var in func.graph.variables}
        ordered_weights = [ref_to_var[captured.ref()] for captured in func.captured_inputs]

    return ordered_weights

def _wrap_graph_def_as_concrete_function(graph_def, func_ref):
    # Note: if input_names is a dictionary (such as `{ts.name: ts.name for ts in example_inputs}`),
    # then the WrappedFunction may occationally have feeding tensors going to the wrong inputs.
    input_names = _get_input_names(func_ref)
    output_names = _get_output_names(func_ref)
    cfunc = wrap_function.function_from_graph_def(graph_def, input_names, output_names)

    # TODO: remove this hack once https://github.com/tensorflow/tensorflow/blob/v2.3.1/tensorflow/python/eager/wrap_function.py#L377 is fixed
    try:
        if cfunc._arg_keywords != func_ref._arg_keywords:
            cfunc._arg_keywords = func_ref._arg_keywords
    except AttributeError:
        pass
    return cfunc

def _wrap_variable_graph_def_as_concrete_function(graph_def, func_ref):
    # Note: if input_names is a dictionary (such as `{ts.name: ts.name for ts in example_inputs}`),
    # then the WrappedFunction may occationally have feeding tensors going to the wrong inputs.
    input_names = _get_all_names(func_ref)
    output_names = _get_output_names(func_ref)
    cfunc = wrap_function.function_from_graph_def(graph_def, input_names, output_names)

    # TODO: remove this hack once https://github.com/tensorflow/tensorflow/blob/v2.3.1/tensorflow/python/eager/wrap_function.py#L377 is fixed
    try:
        if cfunc._arg_keywords != func_ref._arg_keywords:
            cfunc._arg_keywords = func_ref._arg_keywords
    except AttributeError:
        pass
    return cfunc

def _make_keras_model_savable(model, example_inputs):
    if len(example_inputs) == 1:
        example_inputs, = example_inputs
    # hack to propagate metadata for saving
    if hasattr(model, '_set_save_spec'):
        set_save_spec = model._set_save_spec
    elif hasattr(model, '_set_input_attrs'):
        set_save_spec = model._set_input_attrs
    else:
        set_save_spec = None
        logging.warning('Not setting inputs for the traced model {}; you may need to run inference '
                        'before trying to save it'.format(model))
    if set_save_spec is not None:
        set_save_spec(example_inputs)

def analyze_model(func, example_inputs):
    func, example_inputs = _func_to_concrete_func(func, example_inputs)
    graph_def = func.graph.as_graph_def()

    # Get the operators in the model
    operators_in_model = set()
    supported_in_model = set()
    not_supported_operators = set()
    operators_in_model_count = dict()
    total_count = 0
    supported_count = 0

    real_ops = _get_real_ops(graph_def)

    for op in real_ops:
        operators_in_model.add(op)
        operators_in_model_count[op] = operators_in_model_count.get(op, 0) + 1
        total_count += 1
        if op in _SUPPORTED_OPERATOR_TYPES:
            supported_in_model.add(op)
            supported_count += 1
        else:
            not_supported_operators.add(op)

    if len(supported_in_model) > 0:
        print(
            "The following operations are currently supported in tensorflow-neuron for this model:")
        for op in supported_in_model:
            print(op)

    sorted_not_supported_ops = sorted(not_supported_operators)

    if len(sorted_not_supported_ops) > 0:
        print(
            "The following operations are currently not supported in tensorflow-neuron for this model:")
        for op in sorted_not_supported_ops:
            print(op)

    percent_supported = 100.0 * float(supported_count) / float(total_count)
    print("{:.2f}% of all operations ({} of {}) are supported".format(
                percent_supported, supported_count, total_count))

    sorted_all_model_ops = sorted(list(operators_in_model))
    results = {}
    results['percent_supported'] = percent_supported
    results['supported_count'] = supported_count
    results['total_count'] = total_count
    results['supported_operators'] = supported_in_model
    results['unsupported_operators'] = sorted_not_supported_ops
    results['operators'] = sorted_all_model_ops
    results['operator_count'] = operators_in_model_count

    return results


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Sep  9 14:16:05 2020

@author: pageot
"""

import os
import sqlite3
import geopandas as geo
import pandas as pd
import numpy as np
from matplotlib import pyplot as plt
from matplotlib import cm
import csv
import numpy as np
import pandas as pd
import seaborn as sns
import TEST_ANALYSE_SIGNATURE
import shapely.geometry as geom
import descartes
import pickle
import datetime
from scipy import *
from scipy import stats
from pylab import *
from sklearn.metrics import *
from sklearn.linear_model import LinearRegression
from math import log10, cos, sin, asin, sqrt, exp,acos
from collections import namedtuple
import scipy.io
def predict(x):
   return slope * x + intercept

def select_color_date(x):
    couleurs=[]
    for i in range(len(x)):
        if x.date.iloc[i].strftime('%m-%d')<= "04-01" :
            couleurs.append("r")
        else :
            couleurs.append("b")
    return couleurs

    """Calculates reference ET0 based on the Penman-Monteith model.

     This routine calculates the potential evapotranspiration rate from
     a reference crop canopy (ET0) in mm/d. For these calculations the
     analysis by FAO is followed as laid down in the FAO publication
     `Guidelines for computing crop water requirements - FAO Irrigation
     and drainage paper 56 <http://www.fao.org/docrep/X0490E/x0490e00.htm#Contents>`_

    Input variables::

        DAY   -  Python datetime.date object                   -
        LAT   -  Latitude of the site                        degrees
        ELEV  - Elevation above sea level                      m
        TMIN  - Minimum temperature                            C
        TMAX  - Maximum temperature                            C
        AVRAD - Daily shortwave radiation                   J m-2 d-1
        VAP   - 24 hour average vapour pressure               hPa
        WIND2 - 24 hour average windspeed at 2 meter          m/s

    Output is:

        ET0   - Penman-Monteith potential transpiration
                rate from a crop canopy                     [mm/d]
    """
    global DAYL
    # psychrometric instrument constant (kPa/Celsius)
    PSYCON = 0.665
    # albedo and surface resistance [sec/m] for the reference crop canopy
    REFCFC = 0.23; CRES = 70.
    # latent heat of evaporation of water [J/kg == J/mm] and
    LHVAP = 2.45E6
    # Stefan Boltzmann constant (J/m2/d/K4, e.g multiplied by 24*60*60)
    STBC = 4.903E-3
    # Soil heat flux [J/m2/day] explicitly set to zero
    G = 0.

    # mean daily temperature (Celsius)
    TMPA = (TMIN+TMAX)/2.

    # Vapour pressure to kPa
    VAP = hPa2kPa(VAP)
    # atmospheric pressure (kPa)
    T = Celsius2Kelvin(TMPA)
    PATM = 101.3 * pow((T - (0.0065*ELEV))/T, 5.26)

    # psychrometric constant (kPa/Celsius)
    GAMMA = PSYCON * PATM * 1.0E-3

    # Derivative of SVAP with respect to mean temperature, i.e.
    # slope of the SVAP-temperature curve (kPa/Celsius);
    SVAP_TMPA = SatVapourPressure(TMPA)
    DELTA = (4098. * SVAP_TMPA)/pow((TMPA + 237.3), 2)

    # Daily average saturated vapour pressure [kPa] from min/max temperature
    SVAP_TMAX = SatVapourPressure(TMAX)
    SVAP_TMIN = SatVapourPressure(TMIN)
    SVAP = (SVAP_TMAX + SVAP_TMIN) / 2.
    # measured vapour pressure not to exceed saturated vapour pressure
    VAP = np.minimum(VAP, SVAP)

    # Longwave radiation according at Tmax, Tmin (J/m2/d)
    # and preliminary net outgoing long-wave radiation (J/m2/d)
    STB_TMAX = STBC * pow(Celsius2Kelvin(TMAX), 4)
    STB_TMIN = STBC * pow(Celsius2Kelvin(TMIN), 4)
    RNL_TMP = ((STB_TMAX + STB_TMIN) / 2.) * (0.34 - 0.14 * np.sqrt(VAP))

    # Clear Sky radiation [J/m2/DAY] from Angot TOA radiation
    # the latter is found through a call to astro()
    r = astro(doy, LAT, AVRAD)
    CSKYRAD = (0.75 + (2e-05 * ELEV)) * r.ANGOT
    # if CSKYRAD > 0:
        # Final net outgoing longwave radiation [J/m2/day]
    RNL = RNL_TMP * (1.35 * (AVRAD/CSKYRAD) - 0.35)

    # radiative evaporation equivalent for the reference surface
    # [mm/DAY]
    RN = ((1-REFCFC) * AVRAD - RNL)/LHVAP

    # aerodynamic evaporation equivalent [mm/day]
    EA = ((900./(TMPA + 273)) * WIND2 * (SVAP - VAP))

    # Modified psychometric constant (gamma*)[kPa/C]
    MGAMMA = GAMMA * (1. + (CRES/208.*WIND2))

    # Reference ET in mm/day
    ET0 = (DELTA * (RN-G))/(DELTA + MGAMMA) + (GAMMA * EA)/(DELTA + MGAMMA)
        # ET0 = np.maximum(0., ET0)
    # else:
    #     ET0 = np.nan
    return ET0

    """python version of ASTRO routine by Daniel van Kraalingen.

    This subroutine calculates astronomic daylength, diurnal radiation
    characteristics such as the atmospheric transmission, diffuse radiation etc.

    :param day:         date/datetime object
    :param latitude:    latitude of location
    :param radiation:   daily global incoming radiation (J/m2/day)

    output is a `namedtuple` in the following order and tags::

        DAYL      Astronomical daylength (base = 0 degrees)     h
        DAYLP     Astronomical daylength (base =-4 degrees)     h
        SINLD     Seasonal offset of sine of solar height       -
        COSLD     Amplitude of sine of solar height             -
        DIFPP     Diffuse irradiation perpendicular to
                  direction of light                         J m-2 s-1
        ATMTR     Daily atmospheric transmission                -
        DSINBE    Daily total of effective solar height         s
        ANGOT     Angot radiation at top of atmosphere       J m-2 d-1

    Authors: Daniel van Kraalingen
    Date   : April 1991

    Python version
    Author      : Allard de Wit
    Date        : January 2011
    """
    global DAYL
    global ANGOT
    global DAYLP
    global SINLD
    global COSLD
    global DIFPP
    global ATMTR
    global DSINBE
    # Check for range of latitude
    # if abs(latitude) > 90.:
    #     msg = "Latitude not between -90 and 90"
    #     raise RuntimeError(msg)
    LAT = latitude

    # Determine day-of-year (IDAY) from day
    IDAY = doy

    # reassign radiation
    AVRAD = radiation

    # Test if variables for given (day, latitude, radiation) were already calculated
    # in a previous run. If not (e.g. KeyError) calculate the variables, store
    # in cache and return the value.
    # try:
    #     return _cache[(IDAY, LAT, AVRAD)]
    # except KeyError:
    #     pass

    # constants
    RAD = 0.0174533
    PI = 3.1415926
    ANGLE = -4.

    # map python functions to capitals
    SIN = sin
    COS = cos
    ASIN = asin
    REAL = float
    SQRT = np.sqrt
    ABS = abs

    # Declination and solar constant for this day
    DEC = -ASIN(SIN(23.45*RAD)*COS(2.*PI*(REAL(IDAY)+10.)/365.))
    SC  = 1370.*(1.+0.033*COS(2.*PI*REAL(IDAY)/365.))

    # calculation of daylength from intermediate variables
    # SINLD, COSLD and AOB
    SINLD = SIN(RAD*LAT)*SIN(DEC)
    COSLD = COS(RAD*LAT)*COS(DEC)
    AOB = SINLD/COSLD

    # For very high latitudes and days in summer and winter a limit is
    # inserted to avoid math errors when daylength reaches 24 hours in
    # summer or 0 hours in winter.

    # Calculate solution for base=0 degrees
    if abs(AOB) <= 1.0:
        DAYL  = 12.0*(1.+2.*ASIN(AOB)/PI)
        # integrals of sine of solar height
        DSINB  = 3600.*(DAYL*SINLD+24.*COSLD*SQRT(1.-AOB**2)/PI)
        DSINBE = 3600.*(DAYL*(SINLD+0.4*(SINLD**2+COSLD**2*0.5))+
                 12.*COSLD*(2.+3.*0.4*SINLD)*SQRT(1.-AOB**2)/PI)
        ANGOT = SC*DSINB
    else:
        if AOB >  1.0:
            DAYL=24.0
            DSINB = 3600.0*(24.0*SINLD)
            DSINBE = 3600.0*(24.0*(SINLD+0.4*(SINLD**2+COSLD**2*0.5)))
            ANGOT = SC*3600.0*(24.0*SINLD)
        elif AOB < -1.0:
            DAYL=0.0
            DSINB = 3600.0*(0.0*SINLD)
            DSINBE = 3600.0*(0.0*(SINLD+0.4*(SINLD**2+COSLD**2*0.5)))
            ANGOT = SC*3600.0*(0.0*SINLD)
        # integrals of sine of solar height

    # Calculate solution for base=-4 (ANGLE) degrees
    AOB_CORR = (-SIN(ANGLE*RAD)+SINLD)/COSLD
    if abs(AOB_CORR) <= 1.0:
        DAYLP = 12.0*(1.+2.*ASIN(AOB_CORR)/PI)
    elif AOB_CORR > 1.0:
        DAYLP = 24.0
    elif AOB_CORR < -1.0:
        DAYLP = 0.0

    # extraterrestrial radiation and atmospheric transmission
    # ANGOT = SC*DSINB
    # Check for DAYL=0 as in that case the angot radiation is 0 as well
    if DAYL > 0.0:
        ATMTR = AVRAD/ANGOT
    else:
        ATMTR = 0.

    # estimate fraction diffuse irradiation
    if (ATMTR > 0.75):
        FRDIF = 0.23
    elif (ATMTR <= 0.75) and (ATMTR > 0.35):
        FRDIF = 1.33-1.46*ATMTR
    elif (ATMTR <= 0.35) and (ATMTR > 0.07):
        FRDIF = 1.-2.3*(ATMTR-0.07)**2
    else:  # ATMTR <= 0.07
        FRDIF = 1.

    DIFPP = FRDIF*ATMTR*0.5*SC

    # Pack return values in namedtuple, add to cache and return
    astro_nt = namedtuple("AstroResults","DAYL, DAYLP, SINLD, COSLD, DIFPP, "
                                         "ATMTR, DSINBE, ANGOT")
    retvalue = astro_nt(DAYL, DAYLP, SINLD, COSLD, DIFPP, ATMTR, DSINBE, ANGOT)
    _cache[(IDAY, LAT, AVRAD)] = retvalue

    return retvalue

def et0_pm_simple(day1,alt,hmes,lat,tmoy,tmin,tmax,vv,hrmoy,hrmin,hrmax,rs):
    """
    Parameters
    ----------
    day1 : date
        DESCRIPTION.
    alt : Altitude
        DESCRIPTION.
    hmes : Hauteurs de la mesure
        DESCRIPTION.
    lat : latitude (43-lam)
        DESCRIPTION.
    tmoy : Tmepérature moyenne
        DESCRIPTION.
    tmin : Température min
        DESCRIPTION.
    tmax : Température max
        DESCRIPTION.
    vv : vitesse du vent (WS)
        DESCRIPTION.
    hrmoy : humidité relative moyenne
        DESCRIPTION.
    hrmin : humidité relative min
        DESCRIPTION.
    hrmax : humidité relative max
        DESCRIPTION.
    rs : rayonnement solaire
        DESCRIPTION.

    Returns
    -------
    None.

    """
    if ((hrmax-hrmin)<.1 or tmin==tmax):
        return(0)
    try:
        conv_rad=lat * pi / 180.
        u2= vv * 4.87 / log(67.8*hmes-5.42) # hmes ?? dans formule u2 = fait intervenir l'altitude
        rs_mj= rs*24*3600*0.000001
        dr= 1+0.033*cos(2*pi*day1/365)
        d= 0.409*sin((2*pi*day1/365)-1.39)
        ws= acos(-tan(conv_rad)*tan(d))
        ra= (24*60/pi)*0.082*dr*(ws*sin(conv_rad)*sin(d)+cos(conv_rad)*cos(d)*sin(ws))
        rso= ra*(0.75+0.00002*alt)
        es= (0.6108*exp(17.27*tmin/(tmin+237.3))+0.6108*exp(17.27*tmax/(tmax+237.3)))/2
        ea= (hrmin*0.6108*exp(17.27*tmax/(tmax+237.3))+hrmax*0.6108*exp(17.27*tmin/(tmin+237.3)))/(2*100)
        rnl= 4.9*pow(10,-9)*0.5*(pow((tmin+273),4)+pow((tmax+273),4))*(0.34-0.14*sqrt(ea))*(1.35*(rs_mj/rso)-0.35)
        rn=(1-0.23)*rs_mj-rnl
        delta= 4098*0.6108*exp(17.27*tmoy /(tmoy +237.3))/pow((tmoy +237.3),2)
        et0= (0.408*delta*(rn)+(900*0.063/(tmoy+273))*u2*(es-ea))/(delta+0.063*(1+0.34*u2))
        return(et0)
    except:
        return(0)

def et0_pm_simple_kelv(day1,alt,hmes,lat,tmoy,tmin,tmax,vv,hrmoy,hrmin,hrmax,rs):
    """

    """
    if ((hrmax-hrmin)<.1 or tmin==tmax):
        return(0)
    try:
        conv_rad=lat * pi / 180.
        u2= vv * 4.87 / log(67.8*hmes-5.42) # hmes ?? dans formule u2 = fait intervenir l'altitude
        rs_mj= rs*24*3600*0.000001
        dr= 1+0.033*cos(2*pi*day1/365)
        d= 0.409*sin((2*pi*day1/365)-1.39)
        ws= acos(-tan(conv_rad)*tan(d))
        ra= (24*60/pi)*0.082*dr*(ws*sin(conv_rad)*sin(d)+cos(conv_rad)*cos(d)*sin(ws))
        rso= ra*(0.75+0.00002*alt)
        es= (0.6108*exp(17.27*tmin-273.3/(tmin))+0.6108*exp(17.27*tmax-273.3/(tmax)))/2
        ea= (hrmax/100*0.6108*exp(17.27*tmin-273.3/(tmin))+hrmin/100*0.6108*exp(17.27*tmax-273.3/(tmax)))/2
        rnl= 4.9*pow(10,-9)*0.5*(pow((tmin),4)+pow((tmax),4))*(0.34-0.14*sqrt(ea))*(1.35*(rs_mj/rso)-0.35)
        rn=(1-0.23)*rs_mj-rnl
        delta= 4098*0.6108*exp(17.27*tmoy-273.3 /(tmoy ))/pow((tmoy ),2)
        et0= (0.408*delta*(rn)+(900*0.063/(tmoy))*u2*(es-ea))/(delta+0.063*(1+0.34*u2))
        return(et0)
    except:
        return(0)

if __name__ == '__main__':
    d={}
    d["path_labo"]="/datalocal/vboxshare/THESE/BESOIN_EAU/"
    d["path_PC"]="D:/THESE_TMP/"
    d["PC_disk"]="H:/Yann_THESE/BESOIN_EAU/BESOIN_EAU/"
    Celsius2Kelvin = lambda x: x + 273.16
    hPa2kPa = lambda x: x/10.
    # Saturated Vapour pressure [kPa] at temperature temp [C]
    SatVapourPressure = lambda temp: 0.6108 * exp((17.27 * temp) / (237.3 + temp))
# =============================================================================
# Validation modelisation ETR
# =============================================================================
    # Prépartion des datas Eddy-co au format journalière
      # Pour 2017 et autre année

    for y in os.listdir(d["PC_disk"]+"/DONNEES_RAW/PARCELLE_LABO/FLUX_ETR/Corr_fluxes/"):
        print(y)
        # LE_lam=pd.read_csv("H:/Yann_THESE/BESOIN_EAU/BESOIN_EAU/DONNEES_RAW/PARCELLE_LABO/FLUX_ETR/Corr_fluxes/"+y,encoding = 'utf-8',delimiter=",")
        LE_lam=pd.read_csv(d["PC_disk"]+"/DONNEES_RAW/PARCELLE_LABO/FLUX_ETR/Corr_fluxes/"+y,encoding = 'utf-8',delimiter=",")
        LE_lam.drop(0,inplace=True)
        LE_lam["time"]=LE_lam["Time"].astype(str)
        LE_lam["LE_Bowen"]=LE_lam["LE Bowen"].astype(float)
        LE_lam["date"]=LE_lam["time"].apply(lambda x:x[0:10])
        LE_lam["time_hours"]=LE_lam["time"].apply(lambda x:x[10:-3]).replace(":",'.')
        LE_lam["date"]=pd.to_datetime(LE_lam["date"],format="%d/%m/%Y")
        LE_lam.drop(columns=["Time"],inplace=True)
        # Suppression des flux nocture
        LE_lam=LE_lam.loc[(LE_lam.time_hours >=' 06:00') & (LE_lam.time_hours <=' 21:00')]
        LE_lam_day=LE_lam.groupby("date")["LE_Bowen"].mean()
        l=54000/2264000
        ETR_lam_day=LE_lam_day*l
        ETR_lam_day[ETR_lam_day < -1]=pd.NaT
        ETR_lam_day=pd.DataFrame(ETR_lam_day)
        # ETR_lam_day.plot()
        # ETR_lam_day["date"]=LE_lam.date
        ETR_lam_day.to_csv(d["PC_disk"]+"/TRAITEMENT/DATA_VALIDATION/DATA_ETR_CESBIO/DATA_ETR_corr_maize_irri/ETR_maize_irri"+str(y[:-4])+".csv")

        #  Flux non corrigées
    for y in os.listdir(d["PC_disk"]+"/DATA_ETR_CESBIO/DATA_LAM_lec_python/"):
        print(y)
        if "eddy" in y:
            years='2019'
            LE_lam=pd.read_csv(d["PC_disk"]+"/DATA_ETR_CESBIO/DATA_LAM_lec_python/"+str(y),encoding = 'utf-8',delimiter=",")
            LE_lam.drop(0,inplace=True)
            LE_lam["time"]=LE_lam["time"].astype(str)
            LE_lam["LE"]=LE_lam["LE"].astype(float)
            LE_lam["date"]=pd.to_datetime(LE_lam["date"],format="%Y-%m-%d")
            # Suppression des flux nocture
            LE_lam=LE_lam.loc[(LE_lam.time >='06:00') & (LE_lam.time <='21:00')]
            LE_lam_day=LE_lam.groupby("date")["LE"].mean()
            # inter=LE_lam_day*46800
            l=54000/2264000
            ETR_lam_day=LE_lam_day*l
            # ETR_lam_day=LE_lam_day*0.0352
            ETR_lam_day[ETR_lam_day < -1]=pd.NaT
            ETR_lam_day=pd.DataFrame(ETR_lam_day)
            # ETR_lam_day["date"]=LE_lam.date
            ETR_lam_day.to_csv(d["PC_disk"]+"/TRAITEMENT/DATA_VALIDATION/DATA_ETR_CESBIO/DATA_ETR_maize_irri/ETR_maize_irri"+str(years)+".csv")

        else:
            years=y[5:9]
            LE_lam=pd.read_csv(d["PC_disk"]+"/DATA_ETR_CESBIO/DATA_LAM_lec_python/"+str(y),encoding = 'utf-8',delimiter=",")
            LE_lam.drop(0,inplace=True)
            LE_lam["time"]=LE_lam["TIMESTAMP"].astype(str)
            LE_lam["LE"]=LE_lam["LE"].astype(float)
            LE_lam["date"]=LE_lam["time"].apply(lambda x:x[0:10])
            LE_lam["time_hours"]=LE_lam["time"].apply(lambda x:x[10:-3]).replace(":",'.')
            LE_lam["date"]=pd.to_datetime(LE_lam["date"],format="%d/%m/%Y")
            LE_lam.drop(columns=["TIMESTAMP"],inplace=True)
            # Suppression des flux nocture
            LE_lam=LE_lam.loc[(LE_lam.time_hours >=' 06:00') & (LE_lam.time_hours <=' 21:00')]
            LE_lam_day=LE_lam.groupby("date")["LE"].mean()
            l=54000/2264000
            ETR_lam_day=LE_lam_day*l
            # ETR_lam_day=LE_lam_day*0.0352
            ETR_lam_day[ETR_lam_day < -1]=pd.NaT
            ETR_lam_day=pd.DataFrame(ETR_lam_day)
            # ETR_lam_day.plot()
            # ETR_lam_day["date"]=LE_lam.date
            ETR_lam_day.to_csv(d["PC_disk"]+"/TRAITEMENT/DATA_VALIDATION/DATA_ETR_CESBIO/DATA_ETR_maize_irri/ETR_maize_irri"+str(years)+".csv")
#
    for y in ["2008","2010","2012","2014","2015","2019"]:
        ETR_lam_day=pd.read_csv(d["PC_disk"]+"/TRAITEMENT/DATA_VALIDATION/DATA_ETR_CESBIO/DATA_ETR_corr_maize_irri/ETR_maize_irri"+str(y)+".csv")
        ETR_lam_day.date=pd.to_datetime(ETR_lam_day.date,format="%Y-%m-%d",)
        LE_lam_nn_corr=pd.read_csv(d["PC_disk"]+"/TRAITEMENT/DATA_VALIDATION/DATA_ETR_CESBIO/DATA_ETR_maize_irri/ETR_maize_irri"+str(y)+".csv",encoding = 'utf-8',delimiter=",")
        LE_lam_nn_corr.date=pd.to_datetime(LE_lam_nn_corr.date,format="%Y-%m-%d")
        LE_nn_corr=LE_lam_nn_corr.loc[(LE_lam_nn_corr.date >=str(y)+"-04-01") & (LE_lam_nn_corr.date <= str(y)+"-09-30")]
        # plt.scatter(ETR_lam_day.LE_Bowen,LE_nn_corr.LE)
        plt.figure(figsize=(7,7))
        plt.plot(ETR_lam_day.date,ETR_lam_day.LE_Bowen,label='LE_Bowen')
        plt.plot(LE_nn_corr.date,LE_nn_corr.LE,label="LE_nn_corr")
        plt.legend()
        plt.title(str(y))
        plt.savefig(d["PC_disk"]+'/RESULT/PLOT/Comparaison_ETR_lam/plot_LE_corr_vsLE_Bowen_dynamique_'+str(y)+'.png')
        plt.figure(figsize=(7,7))
        plt.scatter(LE_nn_corr.LE,ETR_lam_day.LE_Bowen)
        plt.plot([0.0, 14], [0.0, 14], 'r-', lw=2)
        plt.title(str(y))
        plt.ylabel("ETR corrigés")
        plt.xlabel("ETR non corrigés")
        plt.savefig(d["PC_disk"]+'/RESULT/PLOT/Comparaison_ETR_lam/plot_scatter_LE_corr_vsLE_Bowen_dynamique_large_window'+str(y)+'.png')

        # diff=ETR_lam_day.LE_Bowen.to_list()-LE_nn_corr.LE
        # plt.figure(figsize=(7,7))
        # plt.plot(ETR_lam_day.index,diff)
    ##### mesu diff ratio Bowen et LE nn corr

  # Pour la station de Grignon gestion des LE en ETR
#     for y in ["2019"]:
#         print (y)
#         # df=pd.read_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/DONNEES_RAW/DATA_PARCELLE_GRIGNON/RAW_DATA/DATA_ICOS_ECO/EFDC_L2_Flx_FRGri_"+str(y)+"_30m.txt",sep=',')
#         # df["time"]=df["TIMESTAMP_START"].astype(str)
#         # df["date"]=df["time"].apply(lambda x:x[0:8])
#         # df["time_hours"]=df["time"].apply(lambda x:x[8:]).astype(int)
#         df=pd.read_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/DONNEES_RAW/DATA_PARCELLE_GRIGNON/RAW_DATA/DATA_ICOS_ECO/FR-Gri_Flux_2019.csv")
#         df["time"]=df["TIMESTAMP"].astype(str)
#         df["date"]=df["time"].apply(lambda x:x[0:8])
#         df["time_hours"]=df["time"].apply(lambda x:x[8:])
#         df["date"]=pd.to_datetime(df["date"],format="%Y%m%d")
#         df.drop(columns=["TIMESTAMP"],inplace=True)
#         # Suppression des flux nocture
#         df=df.loc[(df.time_hours >='0600.0') & (df.time_hours <='1900.0')]
#         df=df.groupby('date').mean()
#         df["LE"]=df.eval("LE_1_1_1*0.0352")
#         df[df.LE < -1]=np.nan
#         df.drop(columns=['CO2_1_1_1', 'H2O_1_1_1', 'ZL_1_1_1', 'FC_1_1_1', 'FC_SSITC_TEST',
#         'H_1_1_1', 'H_SSITC_TEST', 'LE_1_1_1', 'LE_SSITC_TEST', 'TAU_1_1_1',
#         'TAU_SSITC_TEST', 'MO_LENGTH_1_1_1', 'USTAR_1_1_1', 'U_SIGMA_1_1_1',
#         'V_SIGMA_1_1_1', 'W_SIGMA_1_1_1', 'WS_1_2_1', 'WD_1_2_1', 'SC_1_1_1'],inplace=True)
#         df.LE.plot()
#         df.to_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/DATA_ETR_CESBIO/DATA_ETR_GRIGNON/DATA_ETR_GRIGNON_ICOS/ETR_GRIGNON"+str(y)+".csv")

# #    Pour 2019
#     ETR_lam=pd.read_csv(d["path_labo"]+"/DATA_ETR_CESBIO/DATA_LAM_lec_python/eddypro_FR-Lam_full_output_2020-01-28T012345_adv.csv")
#     ETR_lam["date"]=pd.to_datetime(ETR_lam["date"],format='%Y-%m-%d')
#     ETR_lam=ETR_lam.loc[(ETR_lam.time >='06:00') & (ETR_lam.time <='19:00')]
#     ETR_lam_day=ETR_lam.groupby("date")["LE"].mean()
#     ETR_lam_day=ETR_lam_day*0.0352
#     ETR_lam_day[ETR_lam_day < -1]=pd.NaT
#     ETR_lam_day.plot()
#     ETR_lam_day.to_csv(d["path_labo"]+"/DATA_ETR_CESBIO/DATA_ETR_LAM/ETR_LAM2019.csv")

#     ## Récuparation date végétation sur ETR
# # =============================================================================
# # Pluvio satation mise en forme
# # =============================================================================
# #  # LAM
#     meteo_lam_station=pd.read_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/DATA_ETR_CESBIO/DATA_LE_LAM_original/eddypro_FR-Lam_biomet_2020-01-28T012345_adv_ss_unit.csv")
#     meteo_lam_station=meteo_lam_station.iloc[:-49]
#     Meteo=meteo_lam_station[["date","P_1_1_1"]]
#     Meteo=Meteo.iloc[1:]
#     Meteo["Prec"]=Meteo.P_1_1_1.astype(float)
#     Meteo[Meteo.Prec<=-9998]=np.nan
#     Meteo["Prec"]=Meteo.Prec*1000
#     Meteo_lam=Meteo.groupby("date").sum()
#     Meteo_lam=Meteo_lam.iloc[:-2]
#     Meteo_lam.to_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/TRAITEMENT/INPUT_DATA/DATA_METEO_BV/PARCELLE_LAM/Meteo_station_2019.csv")

#     # PLUVIO_STAT LAM autres anénes
    # for y in ["2006","2008","2010","2012","2014","2015"]:
    #     mat = scipy.io.loadmat('/datalocal/vboxshare/THESE/BESOIN_EAU/DONNEES_RAW/PARCELLE_LABO/DATA_METEO_LAM/LAM_'+y+'_IS.mat')
    #     Plui=pd.DataFrame(mat["Rain"])
    #     Plui["date"]=pd.to_datetime(np.arange(0,Plui.shape[0]), unit='D',origin=pd.Timestamp(str(y)+'-01-01'))
    #     Plui.columns=["Prec",'date']
    #     Plui.to_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/TRAITEMENT/INPUT_DATA/DATA_METEO_BV/PARCELLE_LAM/Meteo_station_"+y+".csv")
    # df=pd.read_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/DONNEES_RAW/PARCELLE_LABO/DATA_METEO_LAM/LAM_MTO_N3_2005-2016_GP.csv",encoding = 'utf-8',sep=',')
    # df["time"]=df["TEMPS"].astype(str)
    # df["date"]=df["time"].apply(lambda x:x[0:10])
    # # df["time_hours"]=df["time"].apply(lambda x:x[10:])
    # df["Date"]=pd.to_datetime(df["date"],format="%j/%m/%Y")
    # for y in ["2006","2008"]:
    #     a=df.loc[(df.Date >=str(y)+str('-01-01')) & (df.Date <= str(y)+str('-12-31'))]
    #     a.groupby("Date").sum()

    #### Rayonement
    # Rn_lam=meteo_lam_station[["date","time","Rn_1_1_1"]].iloc[1:]
    # # Suppression des flux nocture
    # Rn_lam=Rn_lam.loc[(Rn_lam.time >='06:00') & (Rn_lam.time <='19:00')]
    # Rn_lam=Rn_lam.iloc[1:]
    # Rn_lam["ray"]=Rn_lam.Rn_1_1_1.astype(float)
    # Rn_lam[Rn_lam.ray<=-500]=np.nan
    # Rn_lam_jour=Rn_lam.groupby("date").mean()
    ############ ET0 station
    # meteo_lam_station=meteo_lam_station.iloc[1:]
    # meteo_lam_station["date"]=pd.to_datetime(meteo_lam_station["date"],format='%Y-%m-%d')
    # meteo_lam_station=meteo_lam_station.loc[(meteo_lam_station.time >='06:00') & (meteo_lam_station.time <='19:00')]
    # meteo_lam_station.replace("-9999",np.nan,inplace=True)
    # meteo_lam_station['Ta_1_1_1'] = meteo_lam_station['Ta_1_1_1'].astype(float)
    # meteo_lam_station['Rn_1_1_1'] = meteo_lam_station['Rn_1_1_1'].astype(float)
    # meteo_lam_station['Pa_1_1_1'] = meteo_lam_station['Pa_1_1_1'].astype(float)
    # meteo_lam_station['WS_1_1_1'] = meteo_lam_station['WS_1_1_1'].astype(float)
    # meteo_lam_station['RH_1_1_1'] = meteo_lam_station['RH_1_1_1'].astype(float)
    # meteo_lam_station['DOY'] = meteo_lam_station['DOY'].astype(float)
    # meteo_lam_station["doy"]=round(meteo_lam_station.DOY).astype(int)
    # meteo_lam_station.reset_index(inplace=True)
    # Meteo_lam=meteo_lam_station.groupby("date").mean()
    # Meteo_lam["doy"]=round(Meteo_lam.doy).astype(int)
    # ET0_stat=pd.DataFrame()
    # for i in np.arange(2,366):
    #     Tmax=meteo_lam_station.Ta_1_1_1[meteo_lam_station.doy==i].max()
    #     Tmin=meteo_lam_station.Ta_1_1_1[meteo_lam_station.doy==i].min()
    #     Rhmin=meteo_lam_station.RH_1_1_1[meteo_lam_station.doy==i].min()
    #     Rhmax=meteo_lam_station.RH_1_1_1[meteo_lam_station.doy==i].max()
    #     ET0=et0_pm_simple_kelv(Meteo_lam.doy.iloc[i],5,2,43,Meteo_lam.Ta_1_1_1.iloc[i],Tmin,Tmax,Meteo_lam.WS_1_1_1.iloc[i],Meteo_lam.RH_1_1_1.iloc[i],Rhmin,Rhmax,Meteo_lam.Rn_1_1_1.iloc[i])
    #     print("Michel :%s"%ET0)
    #     ET0_stat=ET0_stat.append([ET0])
    # ET0_stat.reset_index(inplace=True)
    #     ET0_p=penman_monteith(meteo_lam_station.doy.iloc[i],43,5,meteo_lam_station.Ta_1_1_1.iloc[i]-273,meteo_lam_station.Ta_1_1_1.iloc[i]-273.15,meteo_lam_station.Rn_1_1_1.iloc[i],meteo_lam_station.Pa_1_1_1.iloc[i],meteo_lam_station.WS_1_1_1.iloc[i])
    #     # ET0_p=penman_monteith(Meteo_lam.doy.iloc[i],43,5,Meteo_lam.Ta_1_1_1.iloc[i]-273,Meteo_lam.Ta_1_1_1.iloc[i]-273.15,Meteo_lam.Rn_1_1_1.iloc[i],Meteo_lam.Pa_1_1_1.iloc[i],Meteo_lam.WS_1_1_1.iloc[i])
    #     # print(Meteo_lam.doy.iloc[i])
    #     # print(ET0_p)
    #     ET0_stat=ET0_stat.append([ET0_p])
    # ET0_stat.reset_index(inplace=True)
    # ET0_stat["date"]=meteo_lam_station.doy
    # ET0_stati_j=ET0_stat.groupby("date").sum()
    # ETO_SAFRAN=pd.read_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/TRAITEMENT/INPUT_DATA/DATA_METEO_BV/PARCELLE_LAM/meteo_lam_2019.csv")
    # ETO_SAFRAN.date=pd.to_datetime(ETO_SAFRAN.date,format="%Y-%m-%d")
    # ETO_SAFRAN.set_index('date',inplace=True)
    # ETO_SAFRAN.reset_index(inplace=True)
    # ETO_SAFRAN.drop(columns=["Unnamed: 0"],inplace=True)

    # ## Plot
    # Rn_lam_jour.index=pd.to_datetime(Rn_lam_jour.index,format="%Y-%m-%d")
    # Meteo_lam.index=pd.to_datetime(Meteo_lam.index,format="%Y-%m-%d")
    # plt.figure(figsize=(12,10))
    # plt.bar(Meteo_lam.iloc[150:250].index,Meteo_lam.Prec.iloc[150:250],width=1)
    # plt.plot(Rn_lam_jour.iloc[150:250].index,Rn_lam_jour.ray.iloc[150:250],color="r")
    # #### Test détection Irrigation recherche #####
    # # Seuil Rn 150
    # a=Meteo_lam.Prec.where((Rn_lam_jour.ray>350)&(Meteo_lam.Prec>10))
    # a.dropna(inplace=True)
# Gri
    # meteo_lam_station=pd.read_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/DONNEES_RAW/DATA_PARCELLE_GRIGNON/RAW_DATA/DATA_ICOS_ECO/FR-Gri_Meteo_2019.csv")
    # Meteo=meteo_lam_station[["TIMESTAMP","P_1_1_1"]]
    # Meteo["time"]=Meteo["TIMESTAMP"].astype(str)
    # Meteo["date"]=Meteo["time"].apply(lambda x:x[0:8])
    # Meteo["date"]=pd.to_datetime(Meteo["date"],format="%Y%m%d")
    # Meteo.drop(columns=["time","TIMESTAMP"],inplace=True)
    # Meteo["Prec"]=Meteo.P_1_1_1.astype(float)
    # Meteo[Meteo.Prec<=-9998]=np.nan
    # Meteo_Gri=Meteo.groupby("date").sum()
    # Meteo_Gri.drop(columns="P_1_1_1",inplace=True)
    # Meteo_Gri.to_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/TRAITEMENT/INPUT_DATA/DATA_METEO_BV/PARCELLE_GRI/Meteo_station_2019.csv")
# =============================================================================
# Fcover_sigmo
# =============================================================================
    # for i in os.listdir("/datalocal/vboxshare/THESE/BESOIN_EAU/TRAITEMENT/INPUT_DATA/FCOVER_parcelle/FCOVER_Raw/Fcover_sigmo/"):
    #     print(i)
    #     Fcover_sig=pd.read_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/TRAITEMENT/INPUT_DATA/FCOVER_parcelle/FCOVER_Raw/Fcover_sigmo/"+i)
    #     Fcov_sigmo=Fcover_sig.T
    #     Fcov_sigmo=Fcov_sigmo.iloc[2:]
    #     date=pd.date_range(start="2019-01-01",end="2019-12-31")
    #     Fcov_sigmo["date"]=date
    #     Fcov_sigmo.columns=["FCOVER","date"]
    #     Fcov_sigmo.to_csv("/datalocal/vboxshare/THESE/BESOIN_EAU/TRAITEMENT/INPUT_DATA/FCOVER_parcelle/FCOVER_sigmo/"+i)
    #     Fcov_sigmo.plot()
# =============================================================================
# Data Laurent Bi
# =============================================================================
    # for file in os.listdir("/run/media/pageot/PROJET_YP/TS_SWC_NDVI_Lonzee_Yann"):
    #     print(file)
    #     if "NDVI" in file:
    #         df=pd.read_csv("/run/media/pageot/PROJET_YP/TS_SWC_NDVI_Lonzee_Yann/"+file,sep=";")
    #     else:
    #         df=pd.read_csv("/run/media/pageot/PROJET_YP/TS_SWC_NDVI_Lonzee_Yann/"+file)
    #     df["time"]=df["Date"].astype(str)
    #     df["date"]=df["time"].apply(lambda x:x[0:8])
    #     df["date"]=pd.to_datetime(df["date"],format="%d/%m/%y")
    #     df.replace("NaN !",np.nan,inplace=True)
    #     if 'TS' in file:
    #         df["Mean_TS_X-1_float"]=df["Mean_TS_X-1"].astype(float)
    #         df.drop(columns=["Mean_TS_X-1"],inplace=True)
    #     elif 'SWC' in file:
    #         df.replace("#DIV/0 !",np.nan,inplace=True)
    #         df["Mean_SWC_X-4_float"]=df["Mean_SWC_X-4"].astype(float)
    #         df.drop(columns=["Mean_SWC_X-4"],inplace=True)
    #     df=df.groupby("date").mean()
    #     print(df.columns)
    #     df.to_csv("/run/media/pageot/PROJET_YP/TS_SWC_NDVI_Lonzee_Yann/modif_V2_%s.csv"%(file[:-4]))


from datetime import datetime
from statistics import median, mean

from session_tools.writers.markdown_writer import MarkDownWriter

def markdown_analyzer(sessions):
    report_types = set()
    for note_types in [session["to_report"] for session in sessions]:
        for note_type in note_types:
            report_types.add(note_type)
    report_types = list(sorted(report_types))

    report_notes = {}
    for report_type in report_types:
        report_notes[report_type] = list()
    for session in sessions:
        for note in session["session"]:
            if note[1] in report_types:
                report_notes[note[1]].append((session["filename"], note[2]))

    task_breakdown_types = set()
    for task_types in [session["task_breakdown"] for session in sessions]:
        for task_type in task_types:
            task_breakdown_types.add(task_type)
    task_breakdown_types = list(sorted(task_breakdown_types))

    with MarkDownWriter(
        f"analysis-{datetime.now().strftime('%Y%m%dT%H%M%S')}.md"
    ) as md_writer:
        col_width_default = 32
        col_width_wide = 52
        col_width_narrow = 16

        md_writer.header_1("Session analysis")

        # Overview
        md_writer.header_2("Overview")

        # Overview - time spent, number of bugs, issues, questions
        overview_1_columns = [
            ("", col_width_narrow, "L"),
            ("sessions", col_width_narrow, "C"),
            ("planned (min)", col_width_narrow, "C"),
            ("spent (min)", col_width_narrow, "C"),
        ]
        for category in report_types:
            overview_1_columns.append((category, col_width_narrow, "C"))

        overview_1_data = [
            [
                "totals",
                len(sessions),
                sum([int(session["metadata"]["duration"]) for session in sessions]),
                sum(
                    [
                        int(session["metadata"]["actual_duration"])
                        for session in sessions
                    ]
                ),
            ]
        ]

        for notes in report_notes.values():
            overview_1_data[0].append(len(notes))

        md_writer.table(overview_1_columns, overview_1_data)

        # Overview - charters
        overview_2_columns = [
            ("File", col_width_default, "L"),
            ("Tester", col_width_narrow, "L"),
            ("Charter", col_width_wide, "L"),
        ]
        overview_2_data = [
            (
                session["filename"],
                session["metadata"]["tester"],
                session["metadata"]["charter"],
            )
            for session in sessions
        ]
        md_writer.table(overview_2_columns, overview_2_data)

        # The numbers on how time spent, task breakdown
        md_writer.header_2("The numbers")

        numbers_columns = [
            ("File", col_width_default, "L"),
            ("planned (min)", col_width_narrow, "C"),
            ("spent (min)", col_width_narrow, "C"),
        ]
        for category in task_breakdown_types:
            numbers_columns.append((f"{category} (%)", col_width_narrow, "C"))

        numbers_data = []

        for session in sessions:
            session_data = [
                session["filename"],
                session["metadata"]["duration"],
                session["metadata"]["actual_duration"],
            ]

            for category in task_breakdown_types:
                session_data.append(session["task_breakdown"][category])

            numbers_data.append(session_data)

        separator = (
            " -   -   -",
            " -   -   -",
            " -   -   -",
            " -   -   -",
            " -   -   -",
            " -   -   -",
        )
        # ToDo: construct separator based on len(number_of_columns)
        numbers_data.append(separator)

        means = [
            "mean",
            round(mean([int(session["metadata"]["duration"]) for session in sessions])),
            round(
                mean(
                    [
                        int(session["metadata"]["actual_duration"])
                        for session in sessions
                    ]
                )
            ),
        ]
        for category in task_breakdown_types:
            result = round(
                mean([int(session["task_breakdown"][category]) for session in sessions])
            )
            means.append(result)
        numbers_data.append(means)

        medians = [
            "median",
            round(
                median([int(session["metadata"]["duration"]) for session in sessions])
            ),
            round(
                median(
                    [
                        int(session["metadata"]["actual_duration"])
                        for session in sessions
                    ]
                )
            ),
        ]
        for category in task_breakdown_types:
            result = round(
                median(
                    [int(session["task_breakdown"][category]) for session in sessions]
                )
            )
            medians.append(result)
        numbers_data.append(medians)

        md_writer.table(numbers_columns, numbers_data)

        for note_type, notes in report_notes.items():
            md_writer.header_2(f"{note_type.capitalize()}s")
            if len(notes) > 0:
                columns = [
                    ("File", col_width_default, "L"),
                    (note_type, col_width_wide, "L"),
                ]
                md_writer.table(columns, notes)
            else:
                md_writer.add_line(f"No {note_type}s.")


import sys
from io import StringIO
import unittest

class TestClass(unittest.TestCase):
    def assertIO(self, input, output):
        stdout, stdin = sys.stdout, sys.stdin
        sys.stdout, sys.stdin = StringIO(), StringIO(input)
        resolve()
        sys.stdout.seek(0)
        out = sys.stdout.read()[:-1]
        sys.stdout, sys.stdin = stdout, stdin
        self.assertEqual(out, output)

    def test_入力例1(self):
        input = """abcabc
2"""
        output = """3"""
        self.assertIO(input, output)

    def test_入力例2(self):
        input = """aaaaa
1"""
        output = """1"""
        self.assertIO(input, output)

    def test_入力例3(self):
        input = """hello
10"""
        output = """0"""
        self.assertIO(input, output)

def resolve():
  inf = 10**18+1
  # S が短いので全パターン set に投げ込めば良さそう。
  S = input()
  K = int(input())
  ans = set()
  for i in range(len(S) - K + 1):
    ans.add(S[i:i+K])
  print(len(ans))

resolve()

if __name__ == "__main__":
    unittest.main()


from collections import deque

'''2. Написать программу сложения и умножения двух шестнадцатеричных чисел. При этом каждое число представляется как
массив, элементы которого — цифры числа. Например, пользователь ввёл A2 и C4F. Нужно сохранить их как [‘A’,
‘2’] и [‘C’, ‘4’, ‘F’] соответственно. Сумма чисел из примера: [‘C’, ‘F’, ‘1’], произведение - [‘7’, ‘C’, ‘9’, ‘F’,
‘E’]. '''

# декодер
# чтобы получить числовое выражение - .index(str)
# чтобы получить строковой литерал - обращаемся по индексу числового выражения
decoder = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9', 'A', 'B', 'C', 'D', 'E', 'F']
base = 16

def _sum(x, y):
    result = deque()
    higher_order_num = 0  # остаток, который идёт в разряд выше

    if len(x) > len(y):
        x, y = deque(x), deque(y)
    else:
        x, y = deque(y), deque(x)

    while x:
        if y:
            current = decoder.index(x.pop()) + decoder.index(y.pop()) + higher_order_num
        else:
            current = decoder.index(x.pop()) + higher_order_num

        higher_order_num = 0  # сбрасываем остаток, здесь он больше не нужен

        if current < base:
            result.appendleft(decoder[current])
        else:
            result.appendleft(decoder[current - base])
            higher_order_num = 1

    if higher_order_num:
        result.appendleft('1')

    return list(result)

def _mul(x, y):
    result = deque()
    higher_order_num = 0
    spam = deque(deque() for _ in range(len(y)))

    y = deque(y)

    for i in range(len(y)):
        n = decoder.index(y.pop())

        for j in x[::-1]:
            k = decoder.index(j)
            spam[i].appendleft(n * k)

        for _ in range(i):
            spam[i].append(0)  # когда мы умножали столбиком, сдвигали элементы суммирования

    for _ in range(len(spam[-1])):
        res = higher_order_num

        for k in spam:
            if k:
                res += k.pop()

        if res < base:
            result.appendleft(decoder[res])
        else:
            result.appendleft(decoder[res % base])
            higher_order_num = res // base

    if higher_order_num:
        result.appendleft(decoder[higher_order_num])

    return list(result)

a = list(input('Введите первое число: ').upper())
b = list(input('Введите второе число: ').upper())

print(_sum(a, b), 'sum')
print(_mul(a, b), 'multiply')


"""
    This program sends a message to a queue.
    The csv used in this program simulates end of period scores for a night of hockey in the NHL.
    In real life, these messages would be sent at the end of each period. For the sake of this project, messages are sent at 30 second intervals.

    Author: Dylan Eggemeyer
    Date: February 20, 2023

"""
import pika
import sys
import webbrowser
import csv
import time

# Define Global Variables

# decide if you want to show the offer to open RabbitMQ admin site
# Input "True" or "False"
show_offer = "False"

def offer_rabbitmq_admin_site():
    """Offer to open the RabbitMQ Admin website"""
    ans = input("Would you like to monitor RabbitMQ queues? y or n ")
    print()
    if ans.lower() == "y":
        webbrowser.open_new("http://localhost:15672/#/queues")
        print()

def send_message(host: str):
    """
    Creates and sends a message to the queue each execution.
    This process runs and finishes.

    Parameters:
        host (str): the host name or IP address of the RabbitMQ server
    """

    try:
        # create a blocking connection to the RabbitMQ server
        conn = pika.BlockingConnection(pika.ConnectionParameters(host))
        # use the connection to create a communication channel
        ch = conn.channel()
        # use the channel to delete the queue
        ch.queue_delete(queue="hockey_queue")
        # use the channel to declare the durable queue
        ch.queue_declare(queue="hockey_queue", durable=True)

        # open the input file
        input_file = open("hockey_scores.csv", "r")
        # read the input file
        reader = csv.reader(input_file,delimiter=",")
        # skip the headers
        next(reader, None)
        # get message from each row of file
        for row in reader:
            # read a row from the file
            Period, Team1, Team1_Score, Team2, Team2_Score  = row

            # use f string to create message from data
            f_message = f"{Period}, {Team1}, {Team1_Score}, {Team2}, {Team2_Score}"

            # Serialize the message into binary data
            message = f_message.encode()

            # send the message to an individual queue
            ch.basic_publish(exchange="", routing_key="hockey_queue", body=message)

            # print a message to the console for the user
            print(f" [x] Sent {message}")

            # wait 30 seconds before sending next score
            time.sleep(30)
        # close the file
        input_file.close()

    except pika.exceptions.AMQPConnectionError as e:
        print(f"Error: Connection to RabbitMQ server failed: {e}")
        sys.exit(1)
    finally:
        # close the connection to the server
        conn.close()

# Standard Python idiom to indicate main program entry point
# This allows us to import this module and use its functions
# without executing the code below.
# If this is the program being run, then execute the code below
if __name__ == "__main__":
    # show the offer to open Admin Site if show_offer is set to true, else open automatically
    if show_offer == "True":
        offer_rabbitmq_admin_site()
    else:
        webbrowser.open_new("http://localhost:15672/#/queues")
        print()
    # Use the send_message function to start the stream
    send_message("localhost")


import os
import json
import torch
from PIL import Image
from torchvision import transforms
import matplotlib.pyplot as plt

from model import AlexNet

def main():
    device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")
    print("using {} device".format(device))

    data_transform = transforms.Compose(
        [transforms.Resize((224, 224)),
         transforms.ToTensor(),
         transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))]
    )

    base_path = "C:\\Users\\zhangshangyuan\\Desktop\\machine_learning\\AlexNet花朵数据集\\"
    # load_image
    img_path = base_path + "flower\\val\\roses\\1402130395_0b89d76029.jpg"
    assert os.path.exists(img_path), "file '{}' does not exit".format(img_path)
    img = Image.open(img_path)
    plt.imshow(img)

    img = data_transform(img)
    img = torch.unsqueeze(img, dim=0)

    json_path = base_path + "flower\\class_indices.json"
    assert os.path.exists(
        json_path), "file '{}' does not exit".format(json_path)
    json_file = open(json_path, "r")
    # {'0': 'daisy', '1': 'dandelion', '2': 'roses', '3': 'sunflowers', '4': 'tulips'}
    class_index = json.load(json_file)

    # create model and load parameters
    model = AlexNet(num_class=5).to(device)
    weights_path = base_path + 'AlexNet\\checkpoints\\AlexNet_for_flower.pth'
    assert os.path.exists(
        weights_path), "file '{}' does not exit".format(weights_path)
    model.load_state_dict(torch.load(weights_path))

    # infer
    model.eval()
    with torch.no_grad():
        # predict class
        output = torch.squeeze(model(img.to(device))).cpu()
        predict = torch.softmax(output, dim=0)
        predict_class = torch.argmax(predict).numpy()

    print_res = "class: {}   prob: {:.3}".format(class_index[str(predict_class)],
                                                 predict[predict_class].numpy())
    plt.title(print_res)
    for i in range(len(predict)):
        print("class: {:10}   prob: {:.3}".format(class_index[str(i)],
                                                  predict[i].numpy()))
    plt.show()

if __name__ == '__main__':
    main()


import unicodedata
from typing import Union, Tuple

from lxml.etree import _Element

from arxivnlp import xml_match as xm
from arxivnlp.examples.quantities.center import Scalars, ScalarNotation
from arxivnlp.examples.quantities.quantity_kb import Notation, UnitNotation
from arxivnlp.xml_match import LabelTree

class UndesirableMatchException(Exception):
    """
        A match cannot be processed or is rejected, even though it should be a valid match according to the matcher.
        You could say it's a logical error, not a syntax error.
    """
    pass

# ***************
# * BASIC NODES *
# ***************

mrow = xm.tag('mrow')
mo = xm.tag('mo')
mn = xm.tag('mn')
mtext = xm.tag('mtext')
msup = xm.tag('msup')

relational_mo = mo.with_text(
    '[' + ''.join({'=', '≈', '<', '>', '≪', '≫', '≥', '⩾', '≤', '⩽', '∼', '≲', '≳'}) + ']')
empty_tag = xm.any_tag.with_text('^$')
space = mtext.with_text(r'\s*')
base = xm.tag('math') / xm.tag('semantics')

# ***********************
# * NUMBERS AND SCALARS *
# ***********************

def mn_to_number(mn_text: str) -> Union[float, int]:
    reduced = ''
    for c in mn_text:
        if c.isnumeric():
            reduced += c
        elif c == '.':
            reduced += c
        elif c.isspace():
            continue
        else:
            print(f'Can\'t convert mn {repr(mn_text)}')
    return float(reduced) if '.' in reduced else int(reduced)

simple_number = (
                        mn ** 'numeral' |
                        mrow / xm.seq(mo.with_text(r'[-–]') ** 'negative', mn ** 'numeral')
                ) ** 'simplenumber'
power_of_10 = (msup / xm.seq(mn.with_text('^10$'), simple_number ** 'exponent')) ** 'powerof10'
scientific_number = (mrow / xm.seq(simple_number ** 'factor', mtext.with_text('[×]'),
                                   power_of_10)) ** 'scientific'

def tree_to_number(lt: LabelTree) -> Tuple[Union[int, float], str]:
    if lt.label == 'simplenumber':
        sign = -1 if 'negative' in lt else 1
        return sign * mn_to_number(lt['numeral'].node.text), 'simple'
    elif lt.label == 'scientific':
        return tree_to_number(lt['factor'])[0] * tree_to_number(lt['powerof10'])[0], 'scientific'
    elif lt.label == 'powerof10':
        return 10 ** tree_to_number(lt['exponent'])[0], 'powerof10'
    elif len(lt.children) == 1:
        return tree_to_number(lt.children[0])
    raise Exception(f'Unsupported tree: {lt}')

scalar = (simple_number | scientific_number | power_of_10 |
          (mrow / xm.seq(empty_tag, mo.with_text(f'^{unicodedata.lookup("INVISIBLE TIMES")}$'),
                         power_of_10))  # presumably this happens when using siunitx and leaving the factor empty
          ) ** 'scalar'

def scalar_to_scalars(lt: LabelTree) -> Scalars:
    number, type_ = tree_to_number(lt)
    scalar_notation = ScalarNotation(0)
    if type(number) == int:
        scalar_notation |= ScalarNotation.IS_INT
    if type_ == 'scientific':
        scalar_notation |= ScalarNotation.SCIENTIFIC
    return Scalars(float(number), scalar_notation=scalar_notation)

# ***********
# * SYMBOLS *
# ***********

simple_symbol = xm.tag('mi') | xm.tag('mo')
complex_symbol = (
                     (xm.tag('msub') / xm.seq(simple_symbol, xm.any_tag)) |
                     (xm.tag('msup') / xm.seq(simple_symbol, xm.any_tag)) |
                     (xm.tag('msubsup') / xm.seq(simple_symbol, xm.any_tag, xm.any_tag))
)
symbol = simple_symbol | complex_symbol

def symbol_to_notation(node: _Element) -> Notation:
    # TODO: Invisible characters?
    if node.tag in {'mi', 'mn', 'mo'}:
        attr = {'val': node.text}
        if node.tag == 'mi' and node.text and len(node.text) == 1 and node.get('mathvariant') != 'normal':
            attr['isitalic'] = True
        return Notation('i', attr, [])
    elif node.tag in {'msup', 'msub', 'msubsup'}:
        return Notation(node.tag[1:], {}, [symbol_to_notation(child) for child in node.getchildren()])
    elif node.tag == 'mrow':
        # TODO: We need some kind of normalization (merge consecutive identifiers etc.)
        return Notation('seq', {}, [symbol_to_notation(child) for child in node.getchildren()])
    raise Exception('Unsupported symbol node: ' + node.tag)

# *********
# * UNITS *
# *********

simple_unit = (simple_symbol | (xm.tag('msub') / xm.seq(simple_symbol, xm.any_tag))) ** 'simpleunit'
# simple_unit = simple_symbol ** 'simpleunit'

def simple_unit_to_notation(lt: LabelTree) -> Notation:
    assert lt.label == 'simpleunit'
    # TODO: This will get much more complex
    node = lt.node
    return symbol_to_notation(node)

unit_power = (xm.tag('msup') / xm.seq(simple_unit, simple_number ** 'exponent')) ** 'unitpower'
unit_times2 = (mrow / xm.seq(simple_unit | unit_power, xm.maybe(space), simple_unit | unit_power)) ** 'unittimes'
unit_times3 = (mrow / xm.seq(simple_unit | unit_power, xm.maybe(space), simple_unit | unit_power, xm.maybe(space),
                             simple_unit | unit_power)) ** 'unittimes'
unit = (simple_unit | unit_power | unit_times2 | unit_times3) ** 'unit'

def unit_to_unit_notation(lt: LabelTree) -> UnitNotation:
    if lt.label == 'unit':
        assert len(lt.children) == 1
        return unit_to_unit_notation(lt.children[0])
    elif lt.label == 'simpleunit':
        notation = simple_unit_to_notation(lt)
        return UnitNotation([(notation, 1)])
    elif lt.label == 'unitpower':
        notation = simple_unit_to_notation(lt['simpleunit'])
        exponent = tree_to_number(lt['exponent'])[0]
        if exponent not in range(-10, 10):
            raise UndesirableMatchException(f'Bad exponent for a unit: {exponent}')
        return UnitNotation([(notation, exponent)])
    elif lt.label == 'unittimes':
        parts = []
        for subunit in lt.children:
            parts += unit_to_unit_notation(subunit).parts
        return UnitNotation(parts)
    raise Exception(f'Unsupported node: {lt.label}')

# **************
# * QUANTITIES *
# **************

quantity = mrow / xm.seq(scalar, xm.maybe(space), unit)
quantity_in_rel = mrow / xm.seq(xm.maybe(xm.any_tag), relational_mo, quantity)


import os
import argparse
import operator
import numpy as np
from tqdm import tqdm

def get_args():
    parser = argparse.ArgumentParser()
    parser.add_argument('output')
    parser.add_argument('--num_sample', type=int, default=1000)
    parser.add_argument('--min', type=int, default=0)
    parser.add_argument('--max', type=int, default=999)
    parser.add_argument('--num_op', type=int, default=2)

    args = parser.parse_args()
    return args

if __name__=='__main__':
    args = get_args()
    op = [operator.add, operator.sub, operator.mul, operator.ifloordiv]
    str_op = ['+', '-', '*', '/']
    with open(args.output, 'w') as f:
        for i in tqdm(range(args.num_sample)):
            a = np.random.randint(args.min, args.max)
            b = np.random.randint(args.min, args.max)
            o = np.random.randint(0,args.num_op)
            c = op[o](a,b)
            s = str(a)+str(str_op[o])+str(b)+'=^'+str(c)+'$\n'
            f.write(s)


import threading
import json

import webapp2
import httplib2
from oauth2client import crypt, client

_cached_http = httplib2.Http(client.MemoryCache())

def verify_id_token(id_token, audience, http=None,
                    cert_uri=client.ID_TOKEN_VERIFICATON_CERTS):
    """Verifies a signed JWT id_token.

    This function requires PyOpenSSL and because of that it does not work on
    App Engine.

    Args:
      id_token: string, A Signed JWT.
      audience: string, The audience 'aud' that the token should be for.
      http: httplib2.Http, instance to use to make the HTTP request. Callers
        should supply an instance that has caching enabled.
      cert_uri: string, URI of the certificates in JSON format to
        verify the JWT against.

    Returns:
      The deserialized JSON in the JWT.

    Raises:
      oauth2client.crypt.AppIdentityError if the JWT fails to verify.
    """
    if http is None:
        http = _cached_http

    resp, content = http.request(cert_uri)

    if resp.status == 200:
        certs = json.loads(content)
        return crypt.verify_signed_jwt_with_certs(id_token, certs, audience)
    else:
        raise client.VerifyJwtTokenError('Status code: %d' % resp.status)

def get_current_user():
    """  GAE compatibility method. """
    request = webapp2.get_request()
    app = webapp2.get_app()
    audience = app.config.get('idtoken_audience')
    if not audience:
        raise Exception('idtoken_audience not configured')
    token = request.headers.get('x-w69b-idtoken')
    return _user_from_token(token, audience)

class User(object):
    """ Appengine User compatible User class. """

    def __init__(self, email=None, _user_id=None):
        self._email = email
        self._user_id = _user_id

    def email(self):
        return self._email

    def nickname(self):
        return self.email()

    def user_id(self):
        return self._user_id

def _user_from_token(token, audience):
    """ Returns User object or None if token is not valid or None. """
    if token is None:
        return None
    try:
        verified_token = verify_id_token(token, audience)
        return User(email=verified_token['email'],
                    _user_id=verified_token['sub'])
    except (crypt.AppIdentityError, KeyError):
        return None

class Decorator(object):
    """
    Provides user_aware decorator that gives access to user as
    decorator.user (None it not authenticated)

    """

    def __init__(self, audience):
        self._thread_local_store = threading.local()
        self.audience = audience
        self._header_name = 'x-w69b-idtoken'

    @property
    def user(self):
        return getattr(self._thread_local_store, 'user', None)

    @user.setter
    def user(self, user):
        self._thread_local_store.user = user

    def user_aware(self, method):
        """
        Provides decorator.user object that for request handler methods
        that use this decorator. The property is None if no user is
        authenticated.
        """

        def setup_user(request_handler, *args, **kwargs):
            token = request_handler.request.headers.get(self._header_name)
            self.user = _user_from_token(token, self.audience)
            try:
                resp = method(request_handler, *args, **kwargs)
            finally:
                self.user = None
            return resp

        return setup_user


import numpy as np

from ..functional import relu
from ..tensor import Parameter
from .batchnorm import BatchNorm1d
from .linear import Linear
from .module import Module

class _LinearBnActivation1d(Module):
    def __init__(
        self,
        in_features: int,
        out_features: int,
        bias: bool = True,
        compute_mode: str = "default",
        eps=1e-5,
        momentum=0.9,
        affine=True,
        track_running_stats=True,
        **kwargs
    ):
        super().__init__(**kwargs)
        self.out_features = out_features
        self.in_features = in_features
        self.bias = None
        if bias:
            b_shape = (out_features,)
            self.bias = Parameter(np.zeros(b_shape, dtype=np.float32))
        self.linear = Linear(in_features, out_features, bias, compute_mode, **kwargs,)
        self.bn = BatchNorm1d(out_features, eps, momentum, affine, track_running_stats)

class LinearBn1d(_LinearBnActivation1d):
    r"""A fused :class:`~.Module` including :class:`~.module.Linear` and :class:`~.module.BatchNorm1d`.
    Could be replaced with :class:`~.QATModule` version :class:`~.qat.LinearBn1d` using
    :func:`~.quantize.quantize_qat`.
    """

    def forward(self, inp):
        return self.bn(self.linear(inp))

class LinearBnRelu1d(_LinearBnActivation1d):
    r"""A fused :class:`~.Module` including :class:`~.module.Linear`, :class:`~.module.BatchNorm1d` and :func:`~.relu`.
    Could be replaced with :class:`~.QATModule` version :class:`~.qat.LinearBnRelu1d` using :func:`~.quantize.quantize_qat`.
    """

    def forward(self, inp):
        return relu(self.bn(self.linear(inp)))


#!/usr/bin/env python
# encoding: utf-8

"""
@description: 多元线性回归

@version: 1.0
@author: BaoQiang
@license: Apache Licence
@contact: mailbaoqiang@gmail.com
@site: http://www.github.com/githubao
@software: PyCharm
@file: muti_liner.py
@time: 2016/11/23 20:40
"""

from numpy.linalg import inv,lstsq
from numpy import dot,transpose
from sklearn.linear_model import LinearRegression

def main():
    x = [[1,1,1],[1,1,2],[1,2,1]]
    y = [[6],[9],[8]]

    print(dot(inv(dot(transpose(x),x)), dot(transpose(x),y)))

    print(lstsq(x,y)[0])

    model = LinearRegression()
    model.fit(x,y)
    x2 = [[1,3,5]]
    y2 = model.predict(x2)
    print(y2)

if __name__ == '__main__':
    main()