"""
The module implementations in this file are taken from (https://nlp.seas.harvard.edu/2018/04/03/attention.html)
"""
import math
import torch
import torch.nn as nn
import torch.nn.functional as F
from models.transformer.utils import clones, attention

class MultiHeadedAttention(nn.Module):
    def __init__(self, h, d_model, dropout=0.1):
        """
        Implements Figure 2 (right) of the paper (https://arxiv.org/pdf/1706.03762.pdf)
        """
        super(MultiHeadedAttention, self).__init__()
        assert d_model % h == 0
        # We assume d_v always equals d_k
        self.d_k = d_model // h
        self.h = h
        self.linears = clones(nn.Linear(d_model, d_model), 4)
        self.attn = None
        self.dropout = nn.Dropout(p=dropout)

    def forward(self, query, key, value, mask=None):
        """Implements Figure 2"""
        if mask is not None:
            # Same mask applied to all h heads.
            mask = mask.unsqueeze(1)
        nbatches = query.size(0)

        # 1) Do all the linear projections in batch from d_model => h x d_k
        query, key, value = [l(x).view(nbatches, -1, self.h, self.d_k).transpose(1, 2)
                             for l, x in zip(self.linears, (query, key, value))]

        # 2) Apply attention on all the projected vectors in batch.
        x, self.attn = attention(query, key, value, mask=mask, dropout=self.dropout)

        # 3) "Concat" using a view and apply a final linear.
        x = x.transpose(1, 2).contiguous().view(nbatches, -1, self.h * self.d_k)
        return self.linears[-1](x)

class PositionwiseFeedForward(nn.Module):
    """
    Implements FFN equation (Eq2/Page 5) in the paper (https://arxiv.org/pdf/1706.03762.pdf)
    """
    def __init__(self, d_model, d_ff, dropout=0.1):
        super(PositionwiseFeedForward, self).__init__()
        self.w_1 = nn.Linear(d_model, d_ff)
        self.w_2 = nn.Linear(d_ff, d_model)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x):
        return self.w_2(self.dropout(F.relu(self.w_1(x))))

class PositionalEncoding(nn.Module):
    """
    Implement the PE function (Page 6) in the paper (https://arxiv.org/pdf/1706.03762.pdf)
    """
    def __init__(self, d_model, dropout, max_len=5000):
        super(PositionalEncoding, self).__init__()
        self.dropout = nn.Dropout(p=dropout)

        # Compute the positional encodings once in log space.
        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0., max_len).unsqueeze(1)
        div_term = torch.exp(torch.arange(0., d_model, 2) * -(math.log(10000.0) / d_model))
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        pe = pe.unsqueeze(0)
        self.register_buffer('pe', pe)

    def forward(self, x):
        x = x + self.pe[:, :x.size(1)].clone().detach().requires_grad_(False)
        return self.dropout(x)

class LayerNorm(nn.Module):
    """
    Construct a LayerNorm module (See https://arxiv.org/abs/1607.06450 for details)
    """
    def __init__(self, features, eps=1e-6):
        super(LayerNorm, self).__init__()
        self.a_2 = nn.Parameter(torch.ones(features))
        self.b_2 = nn.Parameter(torch.zeros(features))
        self.eps = eps

    def forward(self, x):
        mean = x.mean(-1, keepdim=True)
        std = x.std(-1, keepdim=True)
        return self.a_2 * (x - mean) / (std + self.eps) + self.b_2

class SublayerConnection(nn.Module):
    """
    A residual connection followed by a layer norm.
    Note for code simplicity the norm is first as opposed to last.
    """
    def __init__(self, size, dropout):
        super(SublayerConnection, self).__init__()
        self.norm = LayerNorm(size)
        self.dropout = nn.Dropout(dropout)

    def forward(self, x, sublayer):
        """
        Apply residual connection to any sublayer with the same size.
        """
        return x + self.dropout(sublayer(self.norm(x)))

class EncoderLayer(nn.Module):
    """
    Encoder is made up of self-attn and feed forward
    """
    def __init__(self, size, self_attn, feed_forward, dropout):
        super(EncoderLayer, self).__init__()
        self.self_attn = self_attn
        self.feed_forward = feed_forward
        self.sublayer = clones(SublayerConnection(size, dropout), 2)
        self.size = size

    def forward(self, x, mask):
        """
        Follow Figure 1 (left) for connections [https://arxiv.org/pdf/1706.03762.pdf]
        """
        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, mask))
        return self.sublayer[1](x, self.feed_forward)

class DecoderLayer(nn.Module):
    """
    Decoder is made of self-attn, src-attn, and feed forward
    """
    def __init__(self, size, self_attn, src_attn, feed_forward, dropout):
        super(DecoderLayer, self).__init__()
        self.size = size
        self.self_attn = self_attn
        self.src_attn = src_attn
        self.feed_forward = feed_forward
        self.sublayer = clones(SublayerConnection(size, dropout), 3)

    def forward(self, x, memory, src_mask, tgt_mask):
        """
        Follow Figure 1 (right) for connections [https://arxiv.org/pdf/1706.03762.pdf]
        """
        m = memory
        x = self.sublayer[0](x, lambda x: self.self_attn(x, x, x, tgt_mask))
        x = self.sublayer[1](x, lambda x: self.src_attn(x, m, m, src_mask))
        return self.sublayer[2](x, self.feed_forward)

class Embeddings(nn.Module):
    def __init__(self, d_model, vocab):
        super(Embeddings, self).__init__()
        self.lut = nn.Embedding(vocab, d_model)
        self.d_model = d_model

    def forward(self, x):
        return self.lut(x) * math.sqrt(self.d_model)

class Generator(nn.Module):
    """
    Define standard linear + softmax generation step.
    """
    def __init__(self, d_model, vocab):
        super(Generator, self).__init__()
        self.proj = nn.Linear(d_model, vocab)

    def forward(self, x):
        return F.log_softmax(self.proj(x), dim=-1)


import cv2
from spectral import imshow, view_cube
import numpy as np
import scipy.misc
import matplotlib.pyplot as plt
import matplotlib
import spectral.io.envi as envi

repo_dir = '/home/nodeflux/gits/skripsi'
date_dir = '11042019'
num = '1'

dark_ref = envi.open('{}/datasets/training/{}/dark_ref/capture/dark_ref.hdr'.format(repo_dir, date_dir), '{}/datasets/training/{}/dark_ref/capture/dark_ref.raw'.format(repo_dir, date_dir))
white_ref = envi.open('{}/datasets/training/{}/white_ref/capture/white_ref.hdr'.format(repo_dir, date_dir), '{}/datasets/training/{}/white_ref/capture/white_ref.raw'.format(repo_dir, date_dir))
data_ref = envi.open('{}/datasets/training/{}/bisbul_{}_cut/capture/bisbul_{}_cut.hdr'.format(repo_dir, date_dir, num, num), '{}/datasets/training/{}/bisbul_{}_cut/capture/bisbul_{}_cut.raw'.format(repo_dir, date_dir, num, num))

white_nparr = np.array(white_ref.load())
dark_nparr = np.array(dark_ref.load())
data_nparr = np.array(data_ref.load())

# imshow(data_nparr, (64, 55, 19))

corrected_nparr = np.divide(
    np.subtract(data_nparr, dark_nparr),
    np.subtract(white_nparr, dark_nparr))

corrected_nparr.shape

corrected_nparr = corrected_nparr[120:320, 130:380, :]

# imshow(corrected_nparr, (32, 32, 32))

from numpy import genfromtxt

bands = genfromtxt('{}/datasets/training/helpers/bands.csv'.format(repo_dir), delimiter=',')

leaf_pixel_y = 150
leaf_pixel_x = 150
teflon_pixel_y = 100
teflon_pixel_x = 100

leaf_pixel = corrected_nparr[
    leaf_pixel_y:leaf_pixel_y+1,
    leaf_pixel_x:leaf_pixel_x+1,
    :]
teflon_pixel = corrected_nparr[
    teflon_pixel_y:teflon_pixel_y+1,
    teflon_pixel_x:teflon_pixel_x+1,
    :]

leaf_pixel_squeezed = np.squeeze(leaf_pixel)
teflon_pixel_squeezed = np.squeeze(teflon_pixel)

def hsi2rgb(hsi_nparr, rgb):
    (r, g, b) = rgb
    rgb_nparr = np.dstack((hsi_nparr[:, :, r], hsi_nparr[:, :, g], hsi_nparr[:, :, b]))
    return rgb_nparr

def extract_roi(arr, x, y, w, h):
    roi = arr[y:y+h, x:x+w, :]

    return roi

def draw_bbox(img, x, y, w, h, line):
    cv2.rectangle(img, (x, y), (x+w, y+h), (0, 255, 0), line)
    return img

def extract_rois(coordinates, hsi_nparr, length, line):
    rgb_nparr = hsi2rgb(hsi_nparr, (100, 100, 100))
    img_with_bbox = rgb_nparr
    rois = [] # returned ROIs

    for coordinate in coordinates:
        (x, y) = coordinate
        img_with_bbox = draw_bbox(img_with_bbox, x, y, length, length, line)

        roi = extract_roi(hsi_nparr, x, y, length, length)

        rois.append(roi)

    return rois, img_with_bbox

length = 25 # width and height
line = 2 # bounding box line width

coordinates = [
    (15, 28),
    (100, 23),
    (198, 22),
    (20, 155),
    (114, 148),
    (202, 142)]

rois, bounding_boxed = extract_rois(coordinates, corrected_nparr, length, line)
cv2.imshow('image', bounding_boxed)

# for item in rois:
#     imshow(item, (100, 100, 100))

np.save("{}/datasets/training/{}/bisbul_{}_rois.npy".format(repo_dir, date_dir, num), rois)
np.save("{}/datasets/training/{}/bisbul_{}_corrected.npy".format(repo_dir, date_dir, num), corrected_nparr)

# for i in range(len(rois)):
#     roi = rois[i]
#     intensity = []
#     for b in range(roi.shape[2]):
#         intensity.append(np.mean(roi[:, :, b]))
#     plt.plot(bands, intensity, label='ROI {}'.format(i+1))

# plt.legend(loc='upper left')
# plt.title('Leaf Spectral Footprint\n Mean in ROI Area')
# plt.xlabel('Wavelength (nm)')
# plt.ylabel('Reflectance')
# plt.show()


from django import forms
from models import Topic, Message

"""
class TopicForm(forms.Form):
    title       = forms.CharField()
    category    = forms.HiddenInput()
    forum       = forms.HiddenInput()
    title       = forms.CharField()
    # slug        = forms.CharField(null=True, blank=True)
    sticky      = forms.CheckboxInput()
    user        = forms.HiddenInput()
    text        = forms.CharField(widget=forms.Textarea)
"""

class TopicForm(forms.ModelForm):
    class Meta:
        model = Topic
        exclude = ('user', 'category', 'forum', 'last_user', 'message', 'last_message', 'sticky', 'reply_count',
        'view_count', 'locked', 'active', 'created', 'responded_to', 'modified', 'slug')

class MessageForm(forms.ModelForm):
    class Meta:
        model = Message
        exclude = ('category', 'forum', 'topic', 'user', 'created', 'modified', 'active')


from threading import Event
import errno
import json
try:
    import queue as Queue
except ImportError:
    import Queue

import cephfs
from mgr_module import MgrModule
import orchestrator

from .fs.subvolume import SubvolumePath, SubvolumeClient

class PurgeJob(object):
    def __init__(self, volume_fscid, subvolume_path):
        """
        Purge tasks work in terms of FSCIDs, so that if we process
        a task later when a volume was deleted and recreated with
        the same name, we can correctly drop the task that was
        operating on the original volume.
        """
        self.fscid = volume_fscid
        self.subvolume_path = subvolume_path

class Module(orchestrator.OrchestratorClientMixin, MgrModule):
    COMMANDS = [
        {
            'cmd': 'fs volume ls',
            'desc': "List volumes",
            'perm': 'r'
        },
        {
            'cmd': 'fs volume create '
                   'name=name,type=CephString '
                   'name=size,type=CephString,req=false ',
            'desc': "Create a CephFS volume",
            'perm': 'rw'
        },
        {
            'cmd': 'fs volume rm '
                   'name=vol_name,type=CephString',
            'desc': "Delete a CephFS volume",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolumegroup create '
                   'name=vol_name,type=CephString '
                   'name=group_name,type=CephString ',
            'desc': "Create a CephFS subvolume group in a volume",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolumegroup rm '
                   'name=vol_name,type=CephString '
                   'name=group_name,type=CephString '
                   'name=force,type=CephBool,req=false ',
            'desc': "Delete a CephFS subvolume group in a volume",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolume create '
                   'name=vol_name,type=CephString '
                   'name=sub_name,type=CephString '
                   'name=size,type=CephInt,req=false '
                   'name=group_name,type=CephString,req=false ',
            'desc': "Create a CephFS subvolume in a volume, and optionally, "
                    "with a specific size (in bytes) and in a specific "
                    "subvolume group",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolume rm '
                   'name=vol_name,type=CephString '
                   'name=sub_name,type=CephString '
                   'name=group_name,type=CephString,req=false '
                   'name=force,type=CephBool,req=false ',
            'desc': "Delete a CephFS subvolume in a volume, and optionally, "
                    "in a specific subvolume group",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolume getpath '
                   'name=vol_name,type=CephString '
                   'name=sub_name,type=CephString '
                   'name=group_name,type=CephString,req=false ',
            'desc': "Get the mountpath of a CephFS subvolume in a volume, "
                    "and optionally, in a specific subvolume group",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolumegroup snapshot create '
                   'name=vol_name,type=CephString '
                   'name=group_name,type=CephString '
                   'name=snap_name,type=CephString ',
            'desc': "Create a snapshot of a CephFS subvolume group in a volume",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolumegroup snapshot rm '
                   'name=vol_name,type=CephString '
                   'name=group_name,type=CephString '
                   'name=snap_name,type=CephString '
                   'name=force,type=CephBool,req=false ',
                   'desc': "Delete a snapshot of a CephFS subvolume group in a volume",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolume snapshot create '
                   'name=vol_name,type=CephString '
                   'name=sub_name,type=CephString '
                   'name=snap_name,type=CephString '
                   'name=group_name,type=CephString,req=false ',
            'desc': "Create a snapshot of a CephFS subvolume in a volume, "
                    "and optionally, in a specific subvolume group",
            'perm': 'rw'
        },
        {
            'cmd': 'fs subvolume snapshot rm '
                   'name=vol_name,type=CephString '
                   'name=sub_name,type=CephString '
                   'name=snap_name,type=CephString '
                   'name=group_name,type=CephString,req=false '
                   'name=force,type=CephBool,req=false ',
            'desc': "Delete a snapshot of a CephFS subvolume in a volume, "
                    "and optionally, in a specific subvolume group",
            'perm': 'rw'
        },

        # volume ls [recursive]
        # subvolume ls <volume>
        # volume authorize/deauthorize
        # subvolume authorize/deauthorize

        # volume describe (free space, etc)
        # volume auth list (vc.get_authorized_ids)

        # snapshots?

        # FIXME: we're doing CephFSVolumeClient.recover on every
        # path where we instantiate and connect a client.  Perhaps
        # keep clients alive longer, or just pass a "don't recover"
        # flag in if it's the >1st time we connected a particular
        # volume in the lifetime of this module instance.
    ]

    def __init__(self, *args, **kwargs):
        super(Module, self).__init__(*args, **kwargs)
        self._initialized = Event()

        self._background_jobs = Queue.Queue()

    def serve(self):
        # TODO: discover any subvolumes pending purge, and enqueue
        # them in background_jobs at startup

        # TODO: consume background_jobs
        #   skip purge jobs if their fscid no longer exists

        # TODO: on volume delete, cancel out any background jobs that
        # affect subvolumes within that volume.

        # ... any background init needed?  Can get rid of this
        # and _initialized if not
        self._initialized.set()

    def handle_command(self, inbuf, cmd):
        self._initialized.wait()

        handler_name = "_cmd_" + cmd['prefix'].replace(" ", "_")
        try:
            handler = getattr(self, handler_name)
        except AttributeError:
            return -errno.EINVAL, "", "Unknown command"

        return handler(inbuf, cmd)

    def _pool_base_name(self, volume_name):
        """
        Convention for naming pools for volumes

        :return: string
        """
        return "cephfs.{0}".format(volume_name)

    def _pool_names(self, pool_base_name):
        return pool_base_name + ".meta", pool_base_name + ".data"

    def _cmd_fs_volume_create(self, inbuf, cmd):
        vol_id = cmd['name']
        # TODO: validate name against any rules for pool/fs names
        # (...are there any?)

        size = cmd.get('size', None)

        base_name = self._pool_base_name(vol_id)
        mdp_name, dp_name = self._pool_names(base_name)

        r, outb, outs = self.mon_command({
            'prefix': 'osd pool create',
            'pool': mdp_name,
            'pg_num': 16,
            'pg_num_min': 16,
        })
        if r != 0:
            return r, outb, outs

        # count fs metadata omap at 4x usual rate
        r, outb, outs = self.mon_command({
            'prefix': 'osd pool set',
            'pool': mdp_name,
            'var': "pg_autoscale_bias",
            'val': "4.0",
        })
        if r != 0:
            return r, outb, outs

        r, outb, outs = self.mon_command({
            'prefix': 'osd pool create',
            'pool': dp_name,
            'pg_num': 8
        })
        if r != 0:
            return r, outb, outs

        # Create a filesystem
        # ====================
        r, outb, outs = self.mon_command({
            'prefix': 'fs new',
            'fs_name': vol_id,
            'metadata': mdp_name,
            'data': dp_name
        })

        if r != 0:
            self.log.error("Filesystem creation error: {0} {1} {2}".format(
                r, outb, outs
            ))
            return r, outb, outs

        # TODO: apply quotas to the filesystem root

        # Create an MDS cluster
        # =====================
        spec = orchestrator.StatelessServiceSpec()
        spec.name = vol_id
        try:
            completion = self.add_stateless_service("mds", spec)
            self._orchestrator_wait([completion])
            orchestrator.raise_if_exception(completion)
        except (ImportError, orchestrator.OrchestratorError):
            return 0, "", "Volume created successfully (no MDS daemons created)"
        except Exception as e:
            # Don't let detailed orchestrator exceptions (python backtraces)
            # bubble out to the user
            self.log.exception("Failed to create MDS daemons")
            return -errno.EINVAL, "", str(e)

        return 0, "", ""

    def _volume_get_fs(self, vol_name):
        fs_map = self.get('fs_map')
        for fs in fs_map['filesystems']:
            if fs['mdsmap']['fs_name'] == vol_name:
                return fs

        # Fall through
        return None

    def _volume_get_mds_daemon_names(self, vol_name):
        fs = self._volume_get_fs(vol_name)
        if fs is None:
            return []

        return [i['name'] for i in fs['mdsmap']['info'].values()]

    def _volume_exists(self, vol_name):
        return self._volume_get_fs(vol_name) is not None

    def _cmd_fs_subvolumegroup_create(self, inbuf, cmd):
        """
        :return: a 3-tuple of return code(int), empty string(str), error message (str)
        """
        vol_name = cmd['vol_name']
        group_name = cmd['group_name']

        if not self._volume_exists(vol_name):
            return -errno.ENOENT, "", \
                   "Volume '{0}' not found, create it with `ceph fs volume create` " \
                   "before trying to create subvolume groups".format(vol_name)

        # TODO: validate that subvol size fits in volume size

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            svc.create_group(group_name)

        return 0, "", ""

    def _cmd_fs_subvolumegroup_rm(self, inbuf, cmd):
        """
        :return: a 3-tuple of return code(int), empty string(str), error message (str)
        """
        vol_name = cmd['vol_name']
        group_name = cmd['group_name']

        force = cmd.get('force', False)

        if not self._volume_exists(vol_name):
            if force:
                return 0, "", ""
            else:
                return -errno.ENOENT, "", \
                       "Volume '{0}' not found, cannot remove subvolume group '{0}'".format(vol_name, group_name)

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            # TODO: check whether there are no subvolumes in the group
            try:
                svc.delete_group(group_name)
            except cephfs.ObjectNotFound:
                if force:
                    return 0, "", ""
                else:
                    return -errno.ENOENT, "", \
                           "Subvolume group '{0}' not found, cannot remove it".format(group_name)

        return 0, "", ""

    def _cmd_fs_subvolume_create(self, inbuf, cmd):
        """
        :return: a 3-tuple of return code(int), empty string(str), error message (str)
        """
        vol_name = cmd['vol_name']
        sub_name = cmd['sub_name']

        size = cmd.get('size', None)
        group_name = cmd.get('group_name', None)

        if not self._volume_exists(vol_name):
            return -errno.ENOENT, "", \
                   "Volume '{0}' not found, create it with `ceph fs volume create` " \
                   "before trying to create subvolumes".format(vol_name)

        # TODO: validate that subvol size fits in volume size

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            if group_name and not svc.get_group_path(group_name):
                return -errno.ENOENT, "", \
                    "Subvolume group '{0}' not found, create it with `ceph fs subvolumegroup create` " \
                    "before trying to create subvolumes".format(group_name)
            svp = SubvolumePath(group_name, sub_name)
            svc.create_subvolume(svp, size)

        return 0, "", ""

    def _cmd_fs_subvolume_rm(self, inbuf, cmd):
        """
        :return: a 3-tuple of return code(int), empty string(str), error message (str)
        """
        vol_name = cmd['vol_name']
        sub_name = cmd['sub_name']

        force = cmd.get('force', False)
        group_name = cmd.get('group_name', None)

        fs = self._volume_get_fs(vol_name)
        if fs is None:
            if force:
                return 0, "", ""
            else:
                return -errno.ENOENT, "", \
                       "Volume '{0}' not found, cannot remove subvolume '{1}'".format(vol_name, sub_name)

        vol_fscid = fs['id']

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            if group_name and not svc.get_group_path(group_name):
                if force:
                    return 0, "", ""
                else:
                    return -errno.ENOENT, "", \
                           "Subvolume group '{0}' not found, cannot remove subvolume '{1}'".format(group_name, sub_name)
            svp = SubvolumePath(group_name, sub_name)
            try:
                svc.delete_subvolume(svp)
            except cephfs.ObjectNotFound:
                if force:
                    return 0, "", ""
                else:
                    return -errno.ENOENT, "", \
                           "Subvolume '{0}' not found, cannot remove it".format(sub_name)
            svc.purge_subvolume(svp)

        # TODO: purge subvolume asynchronously
        # self._background_jobs.put(PurgeJob(vol_fscid, svp))

        return 0, "", ""

    def _cmd_fs_volume_rm(self, inbuf, cmd):
        vol_name = cmd['vol_name']

        # Tear down MDS daemons
        # =====================
        try:
            completion = self.remove_stateless_service("mds", vol_name)
            self._orchestrator_wait([completion])
            orchestrator.raise_if_exception(completion)
        except (ImportError, orchestrator.OrchestratorError):
            self.log.warning("OrchestratorError, not tearing down MDS daemons")
        except Exception as e:
            # Don't let detailed orchestrator exceptions (python backtraces)
            # bubble out to the user
            self.log.exception("Failed to tear down MDS daemons")
            return -errno.EINVAL, "", str(e)

        if self._volume_exists(vol_name):
            # In case orchestrator didn't tear down MDS daemons cleanly, or
            # there was no orchestrator, we force the daemons down.
            r, out, err = self.mon_command({
                'prefix': 'fs set',
                'fs_name': vol_name,
                'var': 'cluster_down',
                'val': 'true'
            })
            if r != 0:
                return r, out, err

            for mds_name in self._volume_get_mds_daemon_names(vol_name):
                r, out, err = self.mon_command({
                    'prefix': 'mds fail',
                    'role_or_gid': mds_name})
                if r != 0:
                    return r, out, err

            # Delete CephFS filesystem
            # =========================
            r, out, err = self.mon_command({
                'prefix': 'fs rm',
                'fs_name': vol_name,
                'yes_i_really_mean_it': True,
            })
            if r != 0:
                return r, out, err
        else:
            self.log.warning("Filesystem already gone for volume '{0}'".format(
                vol_name
            ))

        # Delete pools
        # ============
        base_name = self._pool_base_name(vol_name)
        mdp_name, dp_name = self._pool_names(base_name)

        r, out, err = self.mon_command({
            'prefix': 'osd pool rm',
            'pool': mdp_name,
            'pool2': mdp_name,
            'yes_i_really_really_mean_it': True,
        })
        if r != 0:
            return r, out, err

        r, out, err = self.mon_command({
            'prefix': 'osd pool rm',
            'pool': dp_name,
            'pool2': dp_name,
            'yes_i_really_really_mean_it': True,
        })
        if r != 0:
            return r, out, err

        return 0, "", ""

    def _cmd_fs_volume_ls(self, inbuf, cmd):
        fs_map = self.get("fs_map")

        result = []

        for f in fs_map['filesystems']:
            result.append({
                'name': f['mdsmap']['fs_name']
            })

        return 0, json.dumps(result, indent=2), ""

    def _cmd_fs_subvolume_getpath(self, inbuf, cmd):
        vol_name = cmd['vol_name']
        sub_name = cmd['sub_name']

        group_name = cmd.get('group_name', None)

        if not self._volume_exists(vol_name):
            return -errno.ENOENT, "", "Volume '{0}' not found".format(vol_name)

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            if group_name and not svc.get_group_path(group_name):
                return -errno.ENOENT, "", \
                    "Subvolume group '{0}' not found".format(group_name)
            svp = SubvolumePath(group_name, sub_name)
            path = svc.get_subvolume_path(svp)
            if not path:
                return -errno.ENOENT, "", \
                       "Subvolume '{0}' not found".format(sub_name)
        return 0, path, ""

    def _cmd_fs_subvolumegroup_snapshot_create(self, inbuf, cmd):
        vol_name = cmd['vol_name']
        group_name = cmd['group_name']
        snap_name = cmd['snap_name']

        if not self._volume_exists(vol_name):
            return -errno.ENOENT, "", \
                   "Volume '{0}' not found, cannot create snapshot '{1}'".format(vol_name, snap_name)

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            if group_name and not svc.get_group_path(group_name):
                return -errno.ENOENT, "", \
                    "Subvolume group '{0}' not found, cannot create snapshot '{1}'".format(group_name, snap_name)
            svc.create_group_snapshot(group_name, snap_name)

        return 0, "", ""

    def _cmd_fs_subvolumegroup_snapshot_rm(self, inbuf, cmd):
        vol_name = cmd['vol_name']
        group_name = cmd['group_name']
        snap_name = cmd['snap_name']

        force = cmd.get('force', False)

        if not self._volume_exists(vol_name):
            if force:
                return 0, "", ""
            else:
                return -errno.ENOENT, "", \
                       "Volume '{0}' not found, cannot remove subvolumegroup snapshot '{1}'".format(vol_name, snap_name)

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            if group_name and not svc.get_group_path(group_name):
                if force:
                    return 0, "", ""
                else:
                    return -errno.ENOENT, "", \
                           "Subvolume group '{0}' not found, cannot remove subvolumegroup snapshot '{1}'".format(group_name, snap_name)
            try:
                svc.delete_group_snapshot(group_name, snap_name)
            except cephfs.ObjectNotFound:
                if force:
                    return 0, "", ""
                else:
                    return -errno.ENOENT, "", \
                           "Subvolume group snapshot '{0}' not found, cannot remove it".format(sub_name)

        return 0, "", ""

    def _cmd_fs_subvolume_snapshot_create(self, inbuf, cmd):
        vol_name = cmd['vol_name']
        sub_name = cmd['sub_name']
        snap_name = cmd['snap_name']

        group_name = cmd.get('group_name', None)

        if not self._volume_exists(vol_name):
            return -errno.ENOENT, "", \
                   "Volume '{0}' not found, cannot create snapshot '{1}'".format(vol_name, snap_name)

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            if group_name and not svc.get_group_path(group_name):
                return -errno.ENOENT, "", \
                    "Subvolume group '{0}' not found, cannot create snapshot '{1}'".format(group_name, snap_name)
            svp = SubvolumePath(group_name, sub_name)
            if not svc.get_subvolume_path(svp):
                return -errno.ENOENT, "", \
                       "Subvolume '{0}' not found, cannot create snapshot '{1}'".format(sub_name, snap_name)
            svc.create_subvolume_snapshot(svp, snap_name)

        return 0, "", ""

    def _cmd_fs_subvolume_snapshot_rm(self, inbuf, cmd):
        vol_name = cmd['vol_name']
        sub_name = cmd['sub_name']
        snap_name = cmd['snap_name']

        force = cmd.get('force', False)
        group_name = cmd.get('group_name', None)

        if not self._volume_exists(vol_name):
            if force:
                return 0, "", ""
            else:
                return -errno.ENOENT, "", \
                       "Volume '{0}' not found, cannot remove subvolume snapshot '{1}'".format(vol_name, snap_name)

        with SubvolumeClient(self, fs_name=vol_name) as svc:
            if group_name and not svc.get_group_path(group_name):
                if force:
                    return 0, "", ""
                else:
                    return -errno.ENOENT, "", \
                           "Subvolume group '{0}' not found, cannot remove subvolume snapshot '{1}'".format(group_name, snap_name)
            svp = SubvolumePath(group_name, sub_name)
            if not svc.get_subvolume_path(svp):
                if force:
                    return 0, "", ""
                else:
                    return -errno.ENOENT, "", \
                           "Subvolume '{0}' not found, cannot remove subvolume snapshot '{1}'".format(sub_name, snap_name)
            try:
                svc.delete_subvolume_snapshot(svp, snap_name)
            except cephfs.ObjectNotFound:
                if force:
                    return 0, "", ""
                else:
                    return -errno.ENOENT, "", \
                           "Subvolume snapshot '{0}' not found, cannot remove it".format(snap_name)

        return 0, "", ""


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import mock
import tensorflow as tf

from models import model
from models.layers import variables

class ModelTest(tf.test.TestCase):

  def setUp(self):
    self.hparams = tf.contrib.training.HParams(
        learning_rate=0.001,
        decay_rate=0.96,
        decay_steps=1,
        loss_type='softmax')

  class MockModel(model.Model):
    """Implements toy inference method without variable declaration."""

    def inference(self, features):
      logits = tf.constant([[0.1, 0.2, 0.7]])
      return model.Inferred(logits, None)

  class MockModelWithVariable(model.Model):
    """Implements toy inference method with variable declaration."""

    def inference(self, features):
      logits = variables.bias_variable([1, 3])
      return model.Inferred(logits, None)

  @mock.patch.multiple(model.Model, __abstractmethods__=set())
  def testModelInitialization(self):
    """Checks the variables declared in the init method.

      Initialization step should only declare global_step as a non trainable
      variable.
    """
    with tf.Graph().as_default():
      model.Model(self.hparams)
      trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
      self.assertEqual(len(trainable_vars), 0)
      global_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES)
      self.assertEqual(len(global_vars), 1)
      self.assertStartsWith(global_vars[0].name, 'global_step')

  def testSingleTower_WithoutVariable(self):
    """Checks model will raise error when there is no trainable variable."""
    with tf.Graph().as_default():
      test_model = self.MockModel(self.hparams)
      feature = {
          'labels': tf.one_hot([2], 3),
          'num_targets': 1,
      }
      with self.assertRaises(ValueError):
        test_model._single_tower(0, feature)

  def testSingleTower_MultipleCalls(self):
    """Checks the correct variable size after multiple single_tower calls.

      Multiple towers should not declare multiple variables and should share the
      trainable variables. Therefore, variable set size should stay at 1.
      Each tower should have its own operations therefore, total number of
      operations should increase for each tower by the same amount.
    """
    with tf.Graph().as_default() as graph:
      test_model = self.MockModelWithVariable(self.hparams)
      feature = {
          'labels': tf.one_hot([2], 3),
          'num_targets': 1,
      }
      test_model._single_tower(0, feature)
      first_ops = graph.get_operations()
      first_op_num = len(first_ops)
      test_model._single_tower(1, feature)
      second_ops = graph.get_operations()
      second_op_num = len(second_ops)
      test_model._single_tower(2, feature)
      third_ops = graph.get_operations()
      third_op_num = len(third_ops)
      trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
      self.assertEqual(len(trainable_vars), 1)
      self.assertEqual(second_op_num - first_op_num,
                       third_op_num - second_op_num)

  # Model is an abstract class. mock.patch empties the abstractmethods set
  # so that it can be initializable for testing other functions.
  @mock.patch.multiple(model.Model, __abstractmethods__=set())
  def testAverageGradients(self):
    """Checks the correct average for multiple towers and multiple variables.

    The test model has 2 towers with 2 variables shared between them.
    var_0 is getting 1.0 + 3.0 as gradient -> average: 2.0
    var_1 is getting 2.0 + 4.0 as gradient -> average: 3.0
    """
    with tf.Graph().as_default():
      with tf.Session() as session:
        test_model = model.Model(self.hparams)
        grad_0 = tf.constant(1.0)
        grad_1 = tf.constant(2.0)
        tower_0 = [(grad_0, 'var_0'), (grad_1, 'var_1')]
        grad_2 = tf.constant(3.0)
        grad_3 = tf.constant(4.0)
        tower_1 = [(grad_2, 'var_0'), (grad_3, 'var_1')]
        tower_grads = [tower_0, tower_1]
        average_grads = test_model._average_gradients(tower_grads)
        self.assertEqual(len(average_grads), 2)
        self.assertEqual('var_0', average_grads[0][1])
        average_grad_0 = session.run(average_grads[0][0])
        self.assertEqual(2.0, average_grad_0)

        self.assertEqual('var_1', average_grads[1][1])
        average_grad_1 = session.run(average_grads[1][0])
        self.assertEqual(3.0, average_grad_1)

  def testMultiGpu(self):
    """Checks the correct attribute values after multi_gpu call.

    Since tests don't have GPU access, test only covers the correct number of
    elements in the result lists.
    """
    with tf.Graph().as_default():
      test_model = self.MockModelWithVariable(self.hparams)
      feature = {
          'labels': tf.one_hot([2], 3),
          'num_targets': 1,
      }
      test_model.multi_gpu([feature, feature, feature], 3)
      trainable_vars = tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)
      self.assertEqual(len(trainable_vars), 1)

if __name__ == '__main__':
  tf.test.main()


from collections import deque
from DataStructures.Nodes import TreeNode, TrackbackNode

def exploreNeighbors(map, x, y, ocuppiedSymbol, nodeType=TrackbackNode):
	possible_neighbors = list()

	for j, i in [(-1,0), (1,0), (0,-1), (0,1)]:
		if (map[y + i][x + j] != ocuppiedSymbol):
			possible_neighbors.append(nodeType(value=f"{x+j},{y+i}"))
	return possible_neighbors

def checkGoal(actualNode, goalNode):
	if actualNode.value == goalNode.value:
		return True
	return False

def breadthFirstSearch(map, ocuppiedSymbol, initialNode, goalNode):
	tree = initialNode
	actualNode = initialNode
	visitedNodes = dict()
	nextNodes = deque()
	optimalPath = list()
	optimalCost = 0

	while not(checkGoal(actualNode, goalNode)):
		actualNodeX = int(actualNode.value.split(',')[0])
		actualNodeY = int(actualNode.value.split(',')[1])

		neighbors = exploreNeighbors(map, actualNodeX, actualNodeY, ocuppiedSymbol)

		for neighbor in neighbors:
			if not(neighbor.value in visitedNodes):
				neighbor.parent = actualNode
				nextNodes.append(neighbor)

		if len(nextNodes) == 0:
			return None

		actualNode = nextNodes.popleft()

	while actualNode.parent != None:
		optimalCost += 1
		optimalPath.append(actualNode)
		actualNode = actualNode.parent

	optimalPath.append(actualNode)

	optimalPath.reverse()

	return optimalPath, optimalCost

def depthFirstSearch(map, ocuppiedSymbol, initialNode, goalNode):
	actualNode = TrackbackNode(initialNode.value)
	visitedNodes = dict()
	nextNodes = deque()
	optimalPath = list()
	optimalCost = 0

	while not(checkGoal(actualNode, goalNode)):
		actualNodeX = int(actualNode.value.split(',')[0])
		actualNodeY = int(actualNode.value.split(',')[1])

		neighbors = exploreNeighbors(map, actualNodeX, actualNodeY, ocuppiedSymbol)

		for neighbor in neighbors:
			if not(neighbor.value in visitedNodes):
				neighbor.parent = actualNode
				nextNodes.append(neighbor)

		if len(nextNodes) == 0:
			return None

		visitedNodes.update({actualNode.value:None})
		actualNode = nextNodes.pop()

	while actualNode.parent != None:
		optimalCost += 1
		optimalPath.append(actualNode)
		actualNode = actualNode.parent

	optimalPath.append(actualNode)

	optimalPath.reverse()

	return optimalPath, optimalCost

def read_mapfile(filename, initialSymbol, goalSymbol):
	map = list()
	initialNode = None
	goalNode = None

	with open(filename, "r") as reader:
		content = reader.readlines()
		for i, line in enumerate(content):
			line = line.replace("\n", "").replace("\r", "")
			mapLine = list()
			for j, value in enumerate(line):
				mapLine.append(value)
				if value == initialSymbol:
					initialNode = TreeNode(f"{j},{i}")
				elif value == goalSymbol:
					goalNode = TreeNode(f"{j},{i}")
			map.append(mapLine)
	return map, initialNode, goalNode

if __name__ == "__main__":
	goalSymbol = "B"
	initialSymbol = "A"
	ocuppiedSymbol = "#"
	map, initialNode, goalNode = read_mapfile("map1.txt", initialSymbol, goalSymbol)
	print(depthFirstSearch(map, ocuppiedSymbol, initialNode, goalNode))


import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
import sklearn.metrics

from dataclasses import dataclass
from typing import Optional, Iterable

class ModelEvaluator:
    """Helper class to plot evaluation statistics and compare models.
    """

    def __init__(self):
        self._class_results_container = {}
        self._macro_avgs_container = {}
        self._weighted_avgs_container = {}
        self._accuracies_container = {}

        self.classes = None

    def store(self, y_true: np.array, y_pred: np.array, *, name: str) -> None:
        """Stores model performance statistics.

        Args:
            y_true (np.array): The true labels
            y_pred (np.array): The predictions
            name (str): A name for the current model
        """

        classification_report = sklearn.metrics.classification_report(y_true, y_pred, output_dict=True)

        # Pop avg results from classification report
        # so that only results per class remain.
        macro_avg = classification_report.pop('macro avg')
        accuracy = classification_report.pop('accuracy')
        weighted_avg = classification_report.pop('weighted avg')

        if not self.classes:
            self.classes = [key for key, _ in classification_report.items()]

        results_per_class = pd.DataFrame.from_dict(classification_report, orient='index')
        self._class_results_container[name] = results_per_class

        self._macro_avgs_container[name] = macro_avg
        self._weighted_avgs_container[name] = weighted_avg
        self._accuracies_container[name] = accuracy

    @property
    def model_names(self):
        return list(self._class_results_container.keys())

    def get_summary(
        self, y_true: np.array, y_pred: np.array, *,
        name: str,
        classification_report: bool = True,
        store: bool = True
    ) -> None:

        sklearn.metrics.ConfusionMatrixDisplay.from_predictions(y_true, y_pred)

        if name is not None:
            plt.title(name)

        plt.show()

        if classification_report:
            print(sklearn.metrics.classification_report(y_true, y_pred))

        if store:
            self.store(y_true, y_pred, name=name)

    @property
    def class_results(self) -> pd.DataFrame:
        """Results per class for each of the models

        Returns:
            pd.DataFrame: A multiindexed dataframe (name, class)
        """
        return pd.concat(self._class_results_container)

    @property
    def macro_results(self) -> pd.DataFrame:
        """Results per class for each of the models

        Returns:
            pd.DataFrame: A multiindexed dataframe (name, class)
        """
        df = pd.DataFrame(self._macro_avgs_container).T
        df.index = pd.MultiIndex.from_product([df.index, ['macro avgs']])

        return df

    @staticmethod
    def _plot_abstract(
        df, *,
        metric: str,
        savepath: str
    ) -> None:

        df.plot(kind='bar')
        plt.title(metric)
        plt.xticks(rotation=45)
        #plt.legend(title='class')
        if savepath:
            plt.savefig(savepath, bbox_inches='tight')
        plt.show()

    def _get_plot_df(
        self, *,
        type: str,
        metric: str
    ) -> pd.DataFrame:

        transformations = {
            'classwise': self.class_results[metric].unstack(),
            'macro': self.macro_results[metric].unstack().loc[self.model_names,:]
        }

        if type not in transformations.keys():
            allowed_params: str = str(list(transformations.keys()))
            raise ValueError(f'Parameter type must be in {allowed_params}')

        return transformations[type]

    def plot(
        self, *,
        type='classwise',
        metric='f1-score',
        savepath=None
    ) -> None:

        self._plot_abstract(
            df=self._get_plot_df(type=type, metric=metric),
            metric=metric,
            savepath=savepath
        )


import requests
import os

def send_email(name, email, message):

    domain = os.environ["MAILGUN_DOMAIN"]
    api_key = os.environ["MAILGUN_API_KEY"]

    recipient = os.environ["MY_EMAIL"]
    return requests.post(
        f"https://api.mailgun.net/v3/{domain}/messages",
		auth=("api", f"{api_key}"),
		data={
            "from": f"{email}",
			"to": [f"{recipient}"],
			"subject": f"Message From Website: {name}",
			"text": f"{message}"}
  )


# Copyright 2022 The OFA-Sys Team.
# All rights reserved.
# This source code is licensed under the Apache 2.0 license
# found in the LICENSE file in the root directory.

import os
from io import BytesIO

import logging
import warnings
import string

import numpy as np
import torch
import base64
import random
from torchvision import transforms

from PIL import Image, ImageFile

from data import data_utils
from data.ofa_dataset import OFADataset
from utils.vision_helper import RandomAugment
import utils.transforms as T

ImageFile.LOAD_TRUNCATED_IMAGES = True
ImageFile.MAX_IMAGE_PIXELS = None
Image.MAX_IMAGE_PIXELS = None

logger = logging.getLogger(__name__)
warnings.filterwarnings("ignore", "(Possibly )?corrupt EXIF data", UserWarning)

IMAGENET_DEFAULT_MEAN = (0.485, 0.456, 0.406)
IMAGENET_DEFAULT_STD = (0.229, 0.224, 0.225)

def collate(samples, pad_idx, eos_idx):
    if len(samples) == 0:
        return {}

    def merge(key):
        return data_utils.collate_tokens(
            [s[key] for s in samples],
            pad_idx,
            eos_idx=eos_idx,
        )

    id = np.array([s["id"] for s in samples])
    src_tokens = merge("source")
    src_lengths = torch.LongTensor([s["source"].ne(pad_idx).long().sum() for s in samples])

    patch_images = torch.stack([sample['patch_image'] for sample in samples], dim=0)
    patch_masks = torch.cat([sample['patch_mask'] for sample in samples])

    prev_output_tokens = None
    target = None
    if samples[0].get("target", None) is not None:
        target = merge("target")
        tgt_lengths = torch.LongTensor([s["target"].ne(pad_idx).long().sum() for s in samples])
        ntokens = tgt_lengths.sum().item()

        if samples[0].get("prev_output_tokens", None) is not None:
            prev_output_tokens = merge("prev_output_tokens")
    else:
        ntokens = src_lengths.sum().item()

    batch = {
        "id": id,
        "nsentences": len(samples),
        "ntokens": ntokens,
        "net_input": {
            "src_tokens": src_tokens,
            "src_lengths": src_lengths,
            "patch_images": patch_images,
            "patch_masks": patch_masks,
            "prev_output_tokens": prev_output_tokens
        },
        "target": target,
    }

    return batch

class CaptionDataset(OFADataset):
    def __init__(
        self,
        split,
        dataset,
        bpe,
        src_dict,
        tgt_dict=None,
        mimic_t2i=None,
        mimic_i2t=None,
        max_src_length=128,
        max_tgt_length=30,
        patch_image_size=224,
        imagenet_default_mean_and_std=False,
        scst=False,
        negative_dataset=None
    ):
        super().__init__(split, dataset, bpe, src_dict, tgt_dict, mimic_t2i=mimic_t2i, mimic_i2t=mimic_i2t)
        self.max_src_length = max_src_length
        self.max_tgt_length = max_tgt_length
        self.patch_image_size = patch_image_size
        self.scst = scst

        self.transtab = str.maketrans({key: None for key in string.punctuation})

        if imagenet_default_mean_and_std:
            mean = IMAGENET_DEFAULT_MEAN
            std = IMAGENET_DEFAULT_STD
        else:
            mean = [0.5, 0.5, 0.5]
            std = [0.5, 0.5, 0.5]

        scales = np.arange(patch_image_size, patch_image_size+10).tolist()
        """
        # data augmentation
        self.patch_resize_transform = transforms.Compose([
            lambda image: image.convert("RGB"),
            T.RandomResize(scales, max_size=512),
            transforms.CenterCrop(patch_image_size),
            RandomAugment(2, 2, isPIL=True, augs=['Identity', 'AutoContrast', 'Equalize', 'Brightness',
                                                  'ShearX', 'ShearY', 'TranslateX', 'TranslateY', 'Rotate']),
            transforms.ToTensor(),
            transforms.Normalize(mean=mean, std=std),
        ])
        """
        # self.patch_resize_transform = transforms.Compose([
        #     lambda image: image.convert("RGB"),
        #     transforms.Resize((patch_image_size, patch_image_size), interpolation=Image.BICUBIC),
        #     transforms.ToTensor(),
        #     transforms.Normalize(mean=mean, std=std),
        # ])
        # """

        self.patch_resize_transform = transforms.Compose([
            lambda image: image.convert("RGB"),
            transforms.RandomCrop(int(patch_image_size//8*7)),
            transforms.ToTensor(),
            transforms.Normalize(mean=mean, std=std),
        ])

        if type(bpe).__name__ == 'GPT2BPE':
            # self.prompt = " what does the image describe?"
            # self.prompt = " what report can a doctor give for this 2D medical image? "
            # self.prompt = " what can we get from this chest medical image? "
            self.prompt = " what disease does this chest image have? "
        elif type(bpe).__name__ == 'BertBPE':
            self.prompt = "图片描述了什么内容?"
        # print(f"TTTTTTTTTTTTTTTTTTTTTTTTTTTT: ")
        # print(self.pad, self.eos)
        # exit()
        self.negative_dataset = negative_dataset
        self.diseases_mimic = ['Atelectasis', 'Lung Lesion', 'Pneumonia', 'Fracture', 'Cardiomegaly', 'Support Devices', 'Enlarged Cardiomediastinum', 'Pleural Effusion', 'Pleural Other', 'Pneumothorax', 'Consolidation', 'Lung Opacity', 'Edema']

    def __getitem__(self, index):
        # """
        ## random choose from positive or negative dataset (9:1)
        high = 5
        # random_choose = np.random.randint(low=0, high=high)
        random_choose = random.randint(0, high)
        # print(f"self.negative_dataset: {self.negative_dataset}")
        split_ind = 2 if self.negative_dataset is not None else high
        split_ind = high
        # print(f"Random choose: {random_choose, split_ind}")
        if self.split != 'train':
            split_ind = high
        if random_choose <= split_ind:
            uniq_id, image, caption, labels, adjs = self.dataset[index]
            # uniq_id, image, caption, labels, info1, info2 = self.dataset[index]
            # print(f"Use positive data: {uniq_id, image, caption}")
        else:
            uniq_id, image, caption, labels, adjs = self.negative_dataset[index]
            # uniq_id, image, caption, labels, info1, info2 = self.negative_dataset[index]
            # print("Use negative data: {uniq_id, image, captioin}")

        """
        unique_id, image, caption = self.dataset[index]
        # print(unique_id, image, caption)
        """

        caption = caption.lower()
        caption = caption.split('&&')
        caption = [c.strip() for c in caption if len(c.strip())]
        # random.shuffle(caption)
        caption = sorted(caption)
        caption = ", ".join(caption)
        # print(caption)

        images = image.split(',')
        random_image_ind = np.random.randint(low=0, high=len(images))
        # print(f"Random Image: {random_image_ind}")
        image = images[random_image_ind]

        ## add label info in prompt
        # """
        label_list = labels.split('&&')
        label_list = [l.lower() for l in label_list]

        """
        ### add extra info based on classifications
        if 'No Finding'.lower() in label_list:
            extra_info = ' it seems that there are no diseases in this image.'
        else:
            # extra_info = ' there can be {} diseases in this image.'.format(', '.join(label_list))
            extra_info = ' what disease does this image have? there are {} in this image. '.format(', '.join(label_list))
            for d in self.diseases_mimic:
                if d.lower() in label_list:
                    vqa_info = ' does this image have {}? yes, there is {}. '.format(d.lower(), d.lower())
                else:
                    vqa_info = ' does this image have {}? no, no {} find in this image. '.format(d.lower(), d.lower())
                extra_info += vqa_info
        # extra_info = ' test '
        self.prompt = " based on the following info: ' {} ', what can we get from this chest medical image? ".format(extra_info)
        # exit()
        """

        ## Cardiomegaly prompt
        # self.prompt = " based on the following cardiomegaly info: ' {} ', what can we get from this chest medical image? ".format(card_prompt)
        # self.prompt = " what can we get from this chest medical image? "
        # self.prompt = " based on the following pneumothorax info: ' {} ', what can we get from this chest medical image? ".format(card_prompt)

        # self.prompt = ' given the following info: {}, please generate an report for this DR image. '.format(', '.join([info1, info2]))

        # ### 1. just 'what disease does this image have'
        # if 'No Finding'.lower() in label_list:
        #     extra_info = ' it seems that there are no diseases in this image.'
        # else:
        #     extra_info = ' what disease does this image have? there are {} in this image. '.format(', '.join(label_list))
        # self.prompt = " based on the following info: ' {} ', what can we get from this chest medical image? ".format(extra_info)

        # ### 2. 'what disease does this image have?' + 'does this image have xxx ?'
        # if 'No Finding'.lower() in label_list:
        #     extra_info = ' it seems that there are no diseases in this image.'
        # else:
        #     # extra_info = ' there can be {} diseases in this image.'.format(', '.join(label_list))
        #     extra_info = ' what disease does this image have? there are {} in this image. '.format(', '.join(label_list))
        #     for d in self.diseases_mimic:
        #         if d.lower() in label_list:
        #             vqa_info = ' does this image have {}? yes, there is {}. '.format(d.lower(), d.lower())
        #         else:
        #             vqa_info = ' does this image have {}? no, no {} find in this image. '.format(d.lower(), d.lower())
        #         extra_info += vqa_info
        # self.prompt = " based on the following info: ' {} ', what can we get from this chest medical image? ".format(extra_info)

        # ### 3. based on 2, add all prompt
        # prompt_list = prompt.split(',')
        # prompt_dict = {}
        # for i in range(0, len(prompt_list), 2):
        #     if i + 1 >= len(prompt_list):
        #         break
        #     k = prompt_list[i].lower()
        #     v = prompt_list[i+1]
        #     v = ' '.join(v.split('&')).lower()
        #     prompt_dict[k] = v
        # if 'No Finding'.lower() in label_list:
        #     extra_info = ' it seems that there are no diseases in this image.'
        # else:
        #     # extra_info = ' there can be {} diseases in this image.'.format(', '.join(label_list))
        #     extra_info = ' what disease does this image have? there are {} in this image. '.format(', '.join(label_list))
        #     for d in self.diseases_mimic:
        #         cur_disease = d.lower()
        #         if cur_disease in label_list:
        #             adjs = prompt_dict.get(cur_disease, '')
        #             vqa_info = ' does this image have {}? yes, there is {} {}. '.format(cur_disease, adjs, cur_disease)
        #         else:
        #             if cur_disease == 'cardiomegaly':
        #                 vqa_info = ' does this image have {}? no, find normal cardiomegaly in this image. '.format(cur_disease)
        #             else:
        #                 vqa_info = ' does this image have {}? no, no {} find in this image. '.format(cur_disease, cur_disease)
        #         extra_info += vqa_info
        # self.prompt = " based on the following info: ' {} ', what can we get from this chest medical image? ".format(extra_info)

        # ### 4. just 'what disease does this image have' + descriptions
        # if 'No Finding'.lower() in label_list:
        #     extra_info = ' it seems that there are no diseases in this image.'
        # else:
        #     label_with_description = []
        #     for l in label_list:
        #         adjs = prompt_dict.get(l, '')
        #         label_with_description.append(adjs + ' ' + l)
        #     extra_info = ' what disease does this image have? there are {} in this image. '.format(', '.join(label_with_description))
        # self.prompt = " based on the following info: ' {} ', what can we get from this chest medical image? ".format(extra_info)

        # ### For disease level test
        # prompt_list = prompt.split(',')
        # prompt_dict = {}
        # for i in range(0, len(prompt_list), 2):
        #     if i + 1 >= len(prompt_list):
        #         break
        #     k = prompt_list[i].lower()
        #     v = prompt_list[i+1]
        #     v = ' '.join(v.split('&&')).lower()
        #     prompt_dict[k] = v
        # nums = len(prompt_dict)
        # rand_ind = random.randint(0, nums-1)
        # test_k = list(prompt_dict.keys())[rand_ind]
        # test_v = prompt_dict[test_k]
        # self.prompt = " what is the level of {} ?".format(test_k)
        # caption = test_v

        # print(f'************** {self.prompt}')
        # print(f"++++++++++++++++++++ {caption}")

        # self.prompt = " what can we get from this chest medical image? "

        # ### [Classsification]-[location]
        # adj_process = self._process_adjs(adjs)
        # if adj_process is None:
        #     return self.__getitem__(index+1)
        # k, v = adj_process
        # caption = v
        # self.prompt = " where is {} in this DR image ?".format(k)

        # image = Image.open(BytesIO(base64.urlsafe_b64decode(image)))
        if image == '/mnt/lustre/niziyu/data/MIMIC/dr_preprocessed_jpgs/17486231_53979270.jpg':
            return self.__getitem__(index)

        image = Image.open(image)
        patch_image = self.patch_resize_transform(image)
        patch_mask = torch.tensor([True])

        if self.split == 'train' and not self.scst:
            # caption = caption.translate(self.transtab).strip()
            caption_token_list = caption.strip().split()
            tgt_caption = ' '.join(caption_token_list[:self.max_tgt_length])
        else:
            caption = ' '.join(caption.strip().split())
            caption_list = [cap.translate(self.transtab).strip() for cap in caption.strip().split('&&')]
            tgt_caption = '&&'.join(caption_list)
        src_item = self.encode_text(self.prompt)
        tgt_item = self.encode_text(" {}".format(tgt_caption), use_mimic=True)
        src_item = torch.cat([self.bos_item, src_item, self.eos_item])
        target_item = torch.cat([tgt_item, self.eos_item])
        prev_output_item = torch.cat([self.bos_item, tgt_item])

        example = {
            "id": uniq_id,
            "source": src_item,
            "patch_image": patch_image,
            "patch_mask": patch_mask,
            "target": target_item,
            "prev_output_tokens": prev_output_item
        }
        return example

    def collater(self, samples, pad_to_length=None):
        """Merge a list of samples to form a mini-batch.
        Args:
            samples (List[dict]): samples to collate
        Returns:
            dict: a mini-batch containing the data of the task
        """
        return collate(samples, pad_idx=self.pad, eos_idx=self.eos)

    def _process_adjs(self, adjs):
        locations = ['right', 'left', 'mid', 'base', 'lower lobe', 'bibasilar', 'basilar']
        locations = {l:1 for l in locations}
        adj_list = adjs.split(',')
        adj_tuple = []
        for i in range(0, len(adj_list), 2):
            if i + 1 >= len(adj_list):
                continue
            k = adj_list[i].strip().lower()
            vs = adj_list[i+1].split('&')
            vs = sorted([v.strip() for v in vs if v.strip() in locations], reverse=True)
            if not len(vs):
                continue
            adj_tuple.append((k, vs[0]))
        if len(adj_tuple) == 0:
            return None
        return adj_tuple[random.randint(0, len(adj_tuple)-1)]


""" Every function that fit curve for stress-strain related data """

from typing import List, Tuple, Optional
import numpy as np
from scipy.optimize import curve_fit
import matplotlib.pyplot as plt
from mesoplastic import compute

# Every plot will have the same presentation (fontsize, labelsize, ...)
plt.style.use('/home/victor/Documents/stage/data_analysis/presentation.mplstyle')

def flowFit(x: np.ndarray,
            A: float,
            n: float
            ) -> np.ndarray:
    """ Fit function """
    return  A * x**n

def YieldStressFit(x: np.ndarray,
                   sigma_y: float,
                   A: float,
                   n: float
                   ) -> np.ndarray:
    return sigma_y + A * x**n

def FlowCruveFit(list_gdot: np.ndarray,
                 sigma_xy_list: np.ndarray,
                 yield_stress: float,
                 color: str,
                 p0: List[int],
                 vmin: int = 0,
                 vmax: Optional[int] = None
                 ) -> Tuple[np.ndarray, np.ndarray]:
    """ Gets the optimal parameters for the flow curve """
    try:
        popt_flow, pcov_flow = curve_fit(flowFit, list_gdot[vmin:vmax], sigma_xy_list[vmin:vmax], p0)
    except RuntimeError:
        print("Couldn't find the optimal parameters")
        return np.zeros(2, dtype=float), np.zeros((2,2), dtype=float)
    corr = compute.getRsquared(list_gdot[vmin:vmax], sigma_xy_list[vmin:vmax], popt_flow, flowFit)
    plt.loglog(list_gdot[vmin:vmax], flowFit(list_gdot[vmin:vmax], *popt_flow), color=color, linestyle='--')
    print(corr, popt_flow)
    return popt_flow, pcov_flow


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Mon Jul  8 09:33:42 2019

@author: bai
"""

import json
import logging.config
import os
import subModule

def setup_logging(default_path = "logging.json",default_level = logging.INFO,env_key = "LOG_CFG"):
    path = default_path
    value = os.getenv(env_key,None)
    if value:
        path = value
    if os.path.exists(path):
        with open(path,"r") as f:
            config = json.load(f)
            logging.config.dictConfig(config)
    else:
        logging.basicConfig(level = default_level)

def func():

    logging.info("start func")

    logging.info("exec func")

    logging.info("end func")

if __name__ == "__main__":
    logger = logging.getLogger("mainModule")
    setup_logging(default_path = "logging.json")
    subModule.som_function()
    a = subModule.SubModuleClass()
    a.doSomething()
    func()


import cv2
import numpy as np
from tkinter import Tk
from tkinter.filedialog import *

def nothing(x):
    pass;

def main():
    Tk().withdraw

    # Window to set the trackbars for the mask:
    cv2.namedWindow("Tracking")
    # Lower bounds trackbars
    cv2.createTrackbar("LH", "Tracking", 0, 179, nothing)
    cv2.createTrackbar("LS", "Tracking", 0, 255, nothing)
    cv2.createTrackbar("LV", "Tracking", 0, 255, nothing)
    # Upper bounds trackbars
    cv2.createTrackbar("UH", "Tracking", 179, 179, nothing)
    cv2.createTrackbar("US", "Tracking", 255, 255, nothing)
    cv2.createTrackbar("UV", "Tracking", 255, 255, nothing)

    print("ESC key to exit")

    while True:
        # Load the image
        frame = cv2.imread('Template_Matching\Images\Waldo_Map_1.jpg')

        # Convert the image to hsv
        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

        # Get the set trackbar values:
        # Lower Bounds
        l_h = cv2.getTrackbarPos("LH", "Tracking")
        l_s = cv2.getTrackbarPos("LS", "Tracking")
        l_v = cv2.getTrackbarPos("LV", "Tracking")
        # Upper Bounds
        u_h = cv2.getTrackbarPos("UH", "Tracking")
        u_s = cv2.getTrackbarPos("US", "Tracking")
        u_v = cv2.getTrackbarPos("UV", "Tracking")

        # Define the bounds for the color space filter:
        l_mask = np.array([l_h, l_s, l_v])
        u_mask = np.array([u_h, u_s, u_v])

        # Define the mask for the blue:
        mask = cv2.inRange(hsv, l_mask, u_mask)

        # Get the result after the mask is applied:
        result = cv2.bitwise_and(frame, frame, mask=mask)

        # Show the results of the masking:
        cv2.imshow("frame", frame)
        cv2.imshow("mask", mask)
        cv2.imshow("result", result)

        # Wait for the ESC key and close all windows
        key = cv2.waitKey(1)
        if key == 27:
            break
        elif key == 83:
            # saveFilePath  =  asksaveasfilename(initialdir = "/",title = "Mask",filetypes = (("jpeg files","*.jpg"),("all files","*.*")), initialfile = 'mask')
            # cv2.imwrite(saveFilePath, mask)
            saveFilePath  =  asksaveasfilename(initialdir = "/",title = "Result",filetypes = (("jpeg files","*.jpg"),("all files","*.*")), initialfile = 'result')
            cv2.imwrite(saveFilePath, result)
    return 0

if __name__ == "__main__":
    main()
    cv2.destroyAllWindows()


"""
mcpython - a minecraft clone written in python licenced under the MIT-licence
(https://github.com/mcpython4-coding/core)

Contributors: uuk, xkcdjerry (inactive)

Based on the game of fogleman (https://github.com/fogleman/Minecraft), licenced under the MIT-licence
Original game "minecraft" by Mojang Studios (www.minecraft.net), licenced under the EULA
(https://account.mojang.com/documents/minecraft_eula)
Mod loader inspired by "Minecraft Forge" (https://github.com/MinecraftForge/MinecraftForge) and similar

This project is not official by mojang and does not relate to it.
"""
import asyncio
import itertools
import time
import typing

from mcpython import shared
from mcpython.common.world.NetworkSyncedImplementation import NetworkSyncedDimension
from mcpython.engine import logger
from mcpython.engine.network.AbstractPackage import AbstractPackage
from mcpython.engine.network.util import ReadBuffer, WriteBuffer

from .DisconnectionPackage import DisconnectionInitPackage
from .PlayerInfoPackages import PlayerInfoPackage

class DataRequestPackage(AbstractPackage):
    PACKAGE_NAME = "minecraft:world_data_request"

    def __init__(self):
        super().__init__()
        self.request_world_info_state = False
        self.request_player_info_state = False

        self.requested_dimensions = []
        self.requested_chunks = []

    def request_world_info(self):
        self.request_world_info_state = True
        return self

    def request_dimension_info(self, name: str):
        self.requested_dimensions.append(name)
        return self

    def request_chunk(self, dimension: str, cx: int, cz: int):
        # traceback.print_stack()
        self.requested_chunks.append((dimension, cx, cz))
        return self

    def request_player_info(self):
        self.request_player_info_state = True
        return self

    async def write_to_buffer(self, buffer: WriteBuffer):
        buffer.write_bool(self.request_world_info_state)
        buffer.write_bool(self.request_player_info_state)
        await buffer.write_list(self.requested_dimensions, buffer.write_string)
        await buffer.write_list(
            self.requested_chunks,
            lambda e: buffer.write_string(e[0]).write_int(e[1]).write_int(e[1]),
        )

    async def read_from_buffer(self, buffer: ReadBuffer):
        self.request_world_info_state = buffer.read_bool()
        self.request_player_info_state = buffer.read_bool()
        self.requested_dimensions = await buffer.collect_list(buffer.read_string)
        self.requested_chunks = await buffer.collect_list(
            lambda: (buffer.read_string(), buffer.read_int(), buffer.read_int())
        )

    async def handle_inner(self):
        if self.request_world_info_state:
            await self.answer(WorldInfoPackage().setup())

        if self.request_player_info_state:
            await self.answer(PlayerInfoPackage().setup())

        for dimension in self.requested_dimensions:
            await self.answer(DimensionInfoPackage().setup(dimension))

        for dim, cx, cz in self.requested_chunks:
            logger.println(f"collecting chunk information for chunk @{cx}:{cz}@{dim}")
            await self.answer(ChunkDataPackage().setup(dim, (cx, cz)))

class WorldInfoPackage(AbstractPackage):
    """
    Package server -> client for sending requested data back to client.

    Mostly only for sync stuff, but dimensions should be created when needed
    """

    PACKAGE_NAME = "minecraft:world_info"

    def __init__(self):
        super().__init__()
        self.dimensions = []
        self.spawn_point = 0, 0

    def setup(self):
        self.spawn_point = shared.world.spawn_point

        for dim_id, dim in shared.world.dimensions.items():
            self.dimensions.append(
                (dim.get_name(), dim_id, dim.get_world_height_range())
            )

        return self

    async def write_to_buffer(self, buffer: WriteBuffer):
        buffer.write_int(self.spawn_point[0]).write_int(self.spawn_point[1])

        await buffer.write_list(
            self.dimensions,
            lambda e: buffer.write_string(e[0])
            .write_int(e[1])
            .write_int(e[2][0])
            .write_int(e[2][1]),
        )

    async def read_from_buffer(self, buffer: ReadBuffer):
        self.spawn_point = buffer.read_int(), buffer.read_int()

        self.dimensions = await buffer.collect_list(
            lambda: (
                buffer.read_string(),
                buffer.read_int(),
                (buffer.read_int(), buffer.read_int()),
            )
        )

    async def handle_inner(self):
        shared.world.spawn_point = self.spawn_point

        for name, dim_id, height_range in self.dimensions:
            logger.println(f"[NETWORK][WORLD] got dimension info of dimension '{name}'")

            dim = shared.world.get_dimension_by_name(name)

            if not isinstance(dim, NetworkSyncedDimension):
                logger.println(
                    "[NETWORK][WORLD] exchanging for a network sync-ed one..."
                )

                for chunk in dim.chunk_iterator():
                    chunk.hide()

                new_dim = shared.world.dimensions[
                    dim.get_dimension_id()
                ] = NetworkSyncedDimension(
                    shared.world,
                    dim.get_dimension_id(),
                    dim.get_name(),
                    dim.world_generation_config,
                )
                new_dim.world_generation_config_objects = (
                    dim.world_generation_config_objects
                )

                for entity in dim.entity_iterator():
                    entity.dimension = new_dim

            if dim.get_dimension_id() != dim_id:
                await self.answer(
                    DisconnectionInitPackage().set_reason("world dim id miss-match")
                )

            if dim.get_world_height_range() != height_range:
                await self.answer(
                    DisconnectionInitPackage().set_reason("world height miss-match")
                )

class DimensionInfoPackage(AbstractPackage):
    PACKAGE_NAME = "minecraft:dimension_info"

    def __init__(self):
        super().__init__()

    def setup(self, dimension: str):
        return self

class ChunkDataPackage(AbstractPackage):
    PACKAGE_NAME = "minecraft:chunk_data"

    def __init__(self):
        super().__init__()
        self.dimension = "overworld"
        self.position = 0, 0
        self.force = False

        self.blocks = []

    def setup(self, dim: str, position: typing.Tuple[int, int], force=False):
        self.dimension = dim
        self.position = position
        self.force = force

        chunk = shared.world.get_dimension_by_name(dim).get_chunk(position)

        dx, dz = chunk.position[0] * 16, chunk.position[1] * 16

        for x, y, z in itertools.product(range(16), range(256), range(16)):
            x += dx
            z += dz

            b = chunk.get_block((x, y, z), none_if_str=True)
            self.blocks.append(b)

        return self

    async def write_to_buffer(self, buffer: WriteBuffer):
        start = time.time()
        logger.println(
            f"preparing chunk data for chunk @{self.position[0]}:{self.position[1]}@{self.dimension} for networking"
        )
        chunk = shared.world.get_dimension_by_name(self.dimension).get_chunk(
            self.position
        )

        buffer.write_string(self.dimension)
        buffer.write_int(self.position[0]).write_int(self.position[1])
        buffer.write_bool(self.force)

        for b in self.blocks:
            if b is None:
                buffer.write_bool_group([False, False])
            else:
                buffer.write_bool_group([True, chunk.exposed(b.position)])
                buffer.write_string(b.NAME)
                await b.write_to_network_buffer(buffer)

        logger.println(f"-> chunk data ready (took {time.time() - start}s)")

    async def read_from_buffer(self, buffer: ReadBuffer):
        start = time.time()
        logger.println(
            f"preparing chunk data for chunk @{self.position[0]}:{self.position[1]}@{self.dimension} to world"
        )

        self.dimension = buffer.read_string()
        self.position = buffer.read_int(), buffer.read_int()
        self.force = buffer.read_bool()

        for _ in range(16 * 256 * 16):
            is_block, visible = buffer.read_bool_group(2)

            if not is_block:
                self.blocks.append(None)
            else:
                name = buffer.read_string()
                instance = shared.registry.get_by_name("minecraft:block").get(name)()
                await instance.read_from_network_buffer(buffer)
                self.blocks.append((instance, visible))

        logger.println(f"-> chunk data ready (took {time.time() - start}s)")

    async def handle_inner(self):
        start = time.time()
        logger.println(
            f"adding chunk data for chunk @{self.position[0]}:{self.position[1]}@{self.dimension} to world"
        )
        chunk = shared.world.get_dimension_by_name(self.dimension).get_chunk(
            self.position
        )

        if chunk.loaded and not self.force:
            logger.println("-> skipping as chunk exists in game and is loaded")
            return

        dx, dz = self.position

        targets = []
        i = 0
        for x, y, z in itertools.product(
            range(dx * 16, dx * 16 + 16), range(256), range(dz * 16, dz * 16 + 16)
        ):
            block = self.blocks[i]

            if block is not None:
                targets.append(
                    chunk.add_block(
                        (x, y, z),
                        block[0],
                        immediate=False,
                        block_update=False,
                        network_sync=False,
                    )
                )

            i += 1

        await asyncio.gather(*targets)

        chunk.update_all_rendering()

        logger.println(f"-> chunk data fully added (took {time.time()-start}s)")
        chunk.loaded = True

class ChunkBlockChangePackage(AbstractPackage):
    PACKAGE_NAME = "minecraft:chunk_block_update"

    def __init__(self):
        super().__init__()
        self.dimension = None
        self.data = []

    def set_dimension(self, dimension: str):
        self.dimension = dimension
        return self

    def change_position(
        self, position: typing.Tuple[int, int, int], block, update_only=False
    ):
        """
        Updates the block data at a given position in the given dimension

        :param position: the position
        :param block: the block instance
        :param update_only: if to only update the block, not add a new one
        """
        self.data.append((position, block, update_only))
        return self

    async def write_to_network_buffer(self, buffer: WriteBuffer):
        buffer.write_string(self.dimension)

        async def write(e):
            position, block, update_only = e
            buffer.write_long(position[0])
            buffer.write_long(position[1])
            buffer.write_long(position[2])

            if block is None:
                buffer.write_bool_group([False, False])
            else:
                buffer.write_bool_group([True, update_only])
                buffer.write_string(block.NAME)
                await block.write_to_network_buffer(buffer)

        await buffer.write_list(self.data, write)

    async def read_from_network_buffer(self, buffer: ReadBuffer):
        self.dimension = buffer.read_string()

        dimension = shared.world.get_dimension_by_name(self.dimension)

        async def read():
            position = tuple((buffer.read_long() for _ in range(3)))

            is_block, update_only = buffer.read_bool_group(2)

            if not is_block:
                self.data.append((position, None, update_only))
            else:
                name = buffer.read_string()

                if update_only:
                    b = dimension.get_block(position, none_if_str=True)

                    if b is None:
                        logger.println(
                            f"[WARM] got block internal update for block {position} in {self.dimension}, but no block is there!"
                        )
                        return

                    await b.read_from_network_buffer(buffer)

                else:
                    instance = shared.registry.get_by_name("minecraft:block").get(
                        name
                    )()
                    await instance.read_from_network_buffer(buffer)
                    self.data.append((position, instance, update_only))

        self.data = await buffer.collect_list(read)

    async def handle_inner(self):
        dimension = shared.world.get_dimension_by_name(self.dimension)

        targets = []
        for position, block, update_only in self.data:
            targets.append(
                dimension.add_block(
                    position, block, network_sync=False, block_update=False
                )
            )
        await asyncio.gather(*targets)


#自己实现算法的三步

#y = wx + b

import tensorflow as tf
from sklearn.datasets import make_blobs

import numpy as np

import matplotlib.pyplot as plt

data,target = make_blobs(centers=2)
plt.scatter(data[:,0],data[:,1],c=target)
plt.show()

#准备数据，一般加载数据
x = tf.constant(data,dtype=tf.float32)
y = tf.constant(target,dtype=tf.float32)

#初始化参数
W = tf.Variable(np.random.randn(2,1)*0.01 ,dtype=tf.float32)
B = tf.Variable(0. , dtype=tf.float32)
print(W.shape)
#预测函数
def sigmoid(x):
    linear = tf.matmul(x, W) + B
    #return 1 / (1 + tf.exp(-linear))
    return tf.nn.sigmoid(linear)

#损失函数
def cross_entropy_loss(y_true , y_pred):
    y_pred = tf.reshape(y_pred,shape=[100])
    return tf.reduce_mean(-(tf.multiply(y_true , tf.math.log(y_pred)) + tf.multiply((1-y_true),tf.math.log(1-y_pred))))

#训练过程

#定义优化器

optimizer = tf.optimizers.Adam()

def run_optimization():
    with tf.GradientTape() as g:
        y_pred = sigmoid(x)
        loss = cross_entropy_loss(y, y_pred)
        #计算梯度
        gradients = g.gradient(loss,[W,B])
        #更新
        optimizer.apply_gradients(zip(gradients,[W,B]))

def accuracy(y_true , y_pred):
    y_pred = tf.reshape(y_pred ,shape=[100])
    y_ = y_pred.numpy() >0.5
    y_true = y_true.numpy()
    return (y_ == y_true).mean()

for i in range(5000):
    run_optimization()

    if i % 100 == 0:
        y_pred = sigmoid(x)
        acc = accuracy(y , y_pred)
        loss = cross_entropy_loss(y , y_pred)
        print(f'训练次数:{i},准确率：{acc:.3f}，损失：{loss:.4f}')


# -*- coding: utf-8 -*-
"""
Created on Wed Oct 21 14:49:28 2020

@author: Annoy
"""
import scrapy
import json
class QuotesSpider(scrapy.Spider):
    name = "css"
    start_urls = [
        'http://quotes.toscrape.com/page/1/']

    def parse(self, response):
        for quote in response.css('div.quote'):
            filename=(
                'C:\\Users\\Annoy\\Desktop\\Spyder\\Test\\Example\\Example\\spiders\\css-scraper-results.json'
                )
            data=json.dumps({
                    'text': quote.css('span.text::text').get(),
                    'author': quote.css('small.author::text').get(),
                    'tags': quote.css('div.tags a.tag::text').getall()
                })
            with open(filename,'a') as f:
                f.write(data)
        next_page = response.css('li.next a::attr(href)').get()
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse)


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Jan  5 14:20:49 2022

@author: aureoleday
"""

from scipy import signal
import numpy as np
import matplotlib.pyplot as plt
import sympy as sy

sy.init_printing()

class LoopFilter(object):
    def __init__(self, gain, Bn, zeta):
        self.kp = (1/gain)*(4*zeta/(zeta+1/(4*zeta)))*Bn
        self.ki = (1/gain)*(4/(zeta+1/(4*zeta))**2)*(Bn**2)
        self.integrater = 0
        self.lf_out = 0
        print("kp:%f, ki:%f" %(self.kp,self.ki))

    def advance_filter(self, phase_difference):
        self.integrater += self.ki*phase_difference
        self.lf_out = self.integrater + self.kp*phase_difference
        # print(self.lf_out)

    def ef(self):
        return self.lf_out
    def get_p(self):
        return self.kp,self.ki

def cross(arr, th):
    for i in np.arange(arr.size):
        if arr[i] >= th and arr[i+1]<th:
            return i

kd,kp,ki,z = sy.symbols('kd,kp,ki,z')
g = (kp+ki-kp/z)/(1-1/z)
h = 1/(1-1/z)
oltf = sy.cancel(g*h*z**(-1),z)
cltf = sy.cancel(oltf/(1+oltf),z)

T,theta = sy.symbols('T,theta')
hs = theta/(1-1/z)
hr = T*z/(z-1)**2
hp = (T**2/2)*z*(z+1)/(z-1)**3
e_p = sy.cancel(hs*(1-cltf),z)
e_f = sy.cancel(hr*(1-cltf),z)
e_a = sy.cancel(hp*(1-cltf),z)

fvs = sy.limit((z-1)*e_p,z,1)
fvr = sy.limit((z-1)*e_f,z,1)
fvp = sy.limit((z-1)*e_a,z,1)

print("fvs:",fvs)
print("fvr:",fvr)
print("fvp:",fvp)

tf = 0.001
fs = 200000
# #costas
# gain = 0.1
# bn = 0.01
# zeta = 0.707
#main loop
gain = 1
bn = 0.02
zeta = 0.707
print("enbw:", bn*fs*2*np.pi)

plt.close('all')
#main loop
lp = LoopFilter(gain,bn,zeta)
kp,ki = lp.get_p()

z=sy.Symbol('z')
G = (kp+ki-kp/z)/(1-1/z)
H = 1/(1-1/z)
OLTF = sy.cancel(G*H*(z**(-1)))
CLTF = sy.cancel(OLTF/sy.cancel(1+OLTF))

# num_o,den_o = map(lambda x: sy.Poly(x,z).all_coeffs(),sy.fraction(OLTF))
# num_o = np.array(num_o,dtype='float')
# den_o = np.array(den_o,dtype='float')
# sys_o = signal.TransferFunction(num_o,den_o,dt=1/fs)
# w1,H = signal.dlti.freqresp(sys_o)

num,den = map(lambda x: sy.Poly(x,z).all_coeffs(),sy.fraction(CLTF))
num = np.array(num,dtype='float')
den = np.array(den,dtype='float')
print("CLTF:",CLTF)
print("num:",num)
print("den:",den)
print("ts:",-np.log(tf*np.sqrt(1-zeta**2))/(zeta*bn*fs*np.pi))

# num = np.array([kp+ki,-kp])
# den = [1,kp+ki-2,1-kp]
sys = signal.TransferFunction(num,den,dt=1/fs)
# w1,H = signal.dlti.freqresp(sys)
w,mag,phase = signal.dbode(sys,n=10000)
ts,ys = signal.dstep(sys,n=200)
ti,yi = signal.dimpulse(sys,n=200)

po = 10
phase_margin = phase[abs(mag[po:]).argmin()]+180
gain_margin = -mag[abs(phase+180).argmin()]
print("phase_margin:%f,phase_ind:%d\ngain_margin:%f,gain_ind:%d" %(phase_margin,abs(mag[po:]).argmin(),gain_margin,abs(phase+180).argmin()))

plt.close('all')
fig,ax = plt.subplots(2,1,True)
ax[0].semilogx(w,mag,'r')
ax[0].set_ylabel('Gain(dB)')
ax[1].semilogx(w,phase,'b')
ax[1].set_ylabel('Phase(degree)')
ax[1].set_xlabel('Frequency(Hz)')
plt.subplots_adjust(hspace=0)
ax[0].grid(1)
ax[1].grid(1)
plt.show()

fig,ax = plt.subplots(2,1,True)
ax[0].step(ts,np.squeeze(ys),color='r')
ax[0].set_ylabel('Step')
ax[1].step(ti,np.squeeze(yi),color='b')
ax[1].set_ylabel('Impulse')
ax[1].set_xlabel('Time(s)')
plt.subplots_adjust(hspace=0)
ax[0].grid(1)
ax[1].grid(1)
plt.show()

# plt.figure()
# plt.plot(H.real,H.imag,'b')
# plt.plot(H.real,-H.imag,'r')
# plt.scatter(-1,0,color='purple')
# plt.grid(True)


from flask import Flask, render_template
import sqlite3

app = Flask(__name__,static_url_path='/static')

@app.route('/home')
def home():
    create_table()
    insert_table()
    cname = "India"
    p1="Home"
    p2="Contact"
    p3="Services"
    paragraph = "Welcome to Flask World !!"
    return render_template('home.html',cname=cname,p1=p1,p2=p2,p3=p3,paragraph=paragraph)

@app.route('/login')
def login():
    return render_template('login.html')

@app.route('/contact')
def contact():
    return render_template('contact.html')

@app.route('/service')
def service():
    return render_template('service.html')

def create_table():
    conn = sqlite3.connect('infy.db')
    cursor = conn.cursor()

    cursor.execute("""
    CREATE TABLE IF NOT EXISTS emp(
                   id INTEGER,
                   username TEXT,
                   email TEXT

    )
""")
    conn.commit()
    conn.close()

def insert_table():
    conn = sqlite3.connect('infy.db')
    cursor = conn.cursor()
    cursor.execute("INSERT INTO emp(id,username,email) VALUES (?,?,?)",("101","Sahil","sahil@gmail.com"))
    conn.commit()
    conn.close()

app.run(debug=True)


import csv
from flask import Flask, jsonify, request

app = Flask(__name__, static_url_path="")

@app.route("/", methods=["GET"])
def read_csv():
    employee_list = []

    with open('data/employees.csv', 'r') as file_csv:
        content = csv.DictReader(file_csv)

        for employee in content:
            employee_list.append(employee)

    return jsonify(employee_list)

if __name__ == "__main__":
    app.run(debug=True, host="0.0.0.0", port = 5050)


import openpyxl

from classes.TableBlockConstructor import TableBlockConstructor

book = openpyxl.open('CYR_GP.xlsx')
sheet = book.active
# sheet['C3'].value = 'rere'
# sheet['B7'].value = 1
# book.save('rere.xlsx')
years = (2022, 2021, 2020, 2016, 2017)
plan_fact = (40, 40, 40, 5, 5, 0)
fact_plan = (60, 70, 56, 5, 6, 0)

# (self, indicator, event_id, event, main_event_id, main_event, subprogram_id, subprogram,
#                        type, unit, years, plan_marks, fact_marks, comment)

TableBlockConstructor.fill_CYR_sample('x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x')
TableBlockConstructor.fill_GP_sample('x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', 'x', years,
                                     plan_fact, fact_plan, '2005', 'gsfsrhf')


from loguru import logger
from datetime import date
from scripts.utils.attributes.attributes import Faker_
from scripts.utils.attributes.misc import PhoneNumber, Job, Email, KolDok
from scripts.utils.attributes.dates import Year, Month, Day
from scripts.utils.attributes.inn import InnFL, InnYL, Inn
from scripts.utils.attributes.fid import Fid
from scripts.utils.attributes.position import CityName, Region, StreetName, BuildingNumber, BuildingHouse, Quarter, \
    Locality, BirthPlace
from scripts.utils.attributes.fio import LastName, FirstName, MiddleName
from scripts.utils.types.number_abonent import NumberAbonentType
from scripts.utils.types.integer_type import IntegerType, LongType, IntType
from scripts.utils.types.decimal_type import DecimalType
from scripts.utils.types.snils import SnilsType
from scripts.utils.types.string_type import StringType
from scripts.utils.types.data_type import DataType, DataYmdType, DataNType, Date
from scripts.utils.types.inn import InnFLType, InnYLType
from scripts.utils.types.oksm import OKSMType
from scripts.utils.types.test_type import TESTType
from scripts.utils.types.spdul import SPDULType, SPDULschType
from scripts.utils.xml_utils import get_all_value_from_facet

# from scripts.utils.types import Fake_

class Fakers:
    def __init__(self):
        self.all_faker = dict()
        self.all_faker[LastName.name] = LastName()
        self.all_faker[FirstName.name] = FirstName()
        self.all_faker[MiddleName.name] = MiddleName()
        self.all_faker[CityName.name] = CityName()
        self.all_faker[PhoneNumber.name] = PhoneNumber()
        self.all_faker[Region.name] = Region()
        self.all_faker[StreetName.name] = StreetName()
        self.all_faker[BuildingNumber.name] = BuildingNumber()
        self.all_faker[BuildingHouse.name] = BuildingHouse()
        self.all_faker[Quarter.name] = Quarter()
        self.all_faker[Job.name] = Job()
        self.all_faker[Email.name] = Email()
        self.all_faker[Locality.name] = Locality()
        self.all_faker[BirthPlace.name] = BirthPlace()
        self.all_faker[KolDok.name] = KolDok()
        self.all_faker[Month.name] = Month()
        self.all_faker[Day.name] = Day()
        self.all_faker[Year.name] = Year()
        self.all_faker[InnFL.name] = InnFL()  # InnFL, InnYL, Inn
        self.all_faker[InnYL.name] = InnYL()
        self.all_faker[Inn.name] = Inn()
        self.all_faker[Fid.name] = Fid()
        self.all_types = dict()
        self.all_types[NumberAbonentType.name] = NumberAbonentType()
        self.all_types[IntegerType.name] = IntegerType()
        self.all_types[LongType.name] = LongType()
        self.all_types[IntType.name] = IntType()
        self.all_types[DecimalType.name] = DecimalType()
        self.all_types[StringType.name] = StringType()
        self.all_types[SnilsType.name] = SnilsType()
        self.all_types[DataType.name] = DataType()
        self.all_types[Date.name] = Date()
        self.all_types[DataYmdType.name] = DataYmdType()
        self.all_types[DataNType.name] = DataNType()
        self.all_types[InnFLType.name] = InnFLType()
        self.all_types[InnYLType.name] = InnYLType()
        self.all_types[OKSMType.name] = OKSMType()
        self.all_types[TESTType.name] = TESTType()
        self.all_types[SPDULType.name] = SPDULType()
        self.all_types[SPDULschType.name] = SPDULschType()

    def value(self, name, node_type=None, sync_attr=None):
        logger.debug(
            f"name: {name}   node_type: {node_type} node_type.local_name {node_type.local_name if node_type is not None else None}")
        value = None
        merged_types = self.all_facets(node_type)

        if "Пр" not in name and 'Дата' in name and getattr(node_type, "local_name", None) not in ["date", 'ДатаТип',
                                                                                'Дата_ГГГГММДД',
                                                                                'ДатаНТип']:
            logger.trace(f"name: {name}   node_type: {node_type}")
            return self.date_value("%d.%m.%Y")
        elif name in self.all_faker.keys():
            logger.trace(f"name: {name}   node_type: {node_type}")
            value = self.all_faker[name].value(merged_types, sync_attr)
        elif merged_types['name'] in self.all_types:
            logger.trace(f"name: {name}   node_type: {node_type}")
            value = self.all_types[merged_types['name']].value(merged_types, sync_attr)
        elif name in ['СрокДисквЛет', 'СрокДисквМес', 'СрокДисквДн', 'Отправитель', 'ИнвПрич',
                         'Датазаключенияконтракта', 'Датаначала', 'Датаокончания', 'ИНН', 'ИндРейтинг',
                         'ИдЕРН', 'ДатаСвед', 'ПрПодп', 'Код', 'Тип', 'КПП', 'ДатаКонДискв',
                         'ДатаОсвоб', 'ДатаВСилу', 'ДатаАрест', 'ДатаЦиркРоз', 'ДатаИзменРоз', 'Индекс',
                         'КодРегион', 'ДоляПроц']:
            logger.trace(f"name: {name}   node_type: {node_type}")
            value = self.all_types[merged_types["local_name"]].value(merged_types, sync_attr)
        elif "Пр" not in name and 'Дата' in name and getattr(node_type, "local_name", None) != "date":
            logger.trace(f"name: {name}   node_type: {node_type}")
            value = self.date_value("%d.%m.%Y")
        return value

    @staticmethod
    def date_value(pattern):
        return Faker_._fake.date_between_dates(date_start=date(1900, 1, 1), date_end=date(2099, 12, 31)).strftime(
            pattern)

    @staticmethod
    def all_facets(node_type):
        # value = Fake_.numerify(text=f"{'#'*{facts_values['length']}")
        all_facets_types = [item.split("}")[1] for item in node_type.facets if item is not None]
        logger.trace(f" node_type: {node_type}")
        type_ = dict()
        node_name = "node_type"
        type_[node_name] = Fakers.all_facets_fot_type(node_name, node_type)
        logger.trace(f"node_name {node_name}")
        node_name = "base_type"
        type_[node_name] = Fakers.get_facets(node_name, node_type)
        logger.trace(f"node_name {node_name}")
        node_name = "primitive_type"
        type_[node_name] = Fakers.get_facets(node_name, node_type)
        logger.trace(f"node_name {node_name}")
        node_name = "member_types"
        type_[node_name] = Fakers.get_facets(node_name, node_type)
        logger.trace(f"node_name {node_name}")

        # Объеденить типы
        merged_type = dict()
        for key in type_["node_type"]:
            if type_["node_type"][key] is not None:
                merged_type[key] = type_["node_type"][key]
            elif type_["primitive_type"] is not None and type_["primitive_type"].get(key, None) is not None:
                merged_type[key] = type_["primitive_type"][key]
            elif type_["base_type"] is not None and type_["base_type"].get(key, None) is not None:
                merged_type[key] = type_["base_type"][key]
            elif type_["member_types"] is not None and type_["member_types"].get(key, None) is not None:
                merged_type[key] = type_["member_types"][key]
            else:
                merged_type[key] = None
        Fakers.add_keys(merged_type, type_, "primitive_type")
        Fakers.add_keys(merged_type, type_, "base_type")
        Fakers.add_keys(merged_type, type_, "member_types")

        if merged_type["local_name"] not in ["string", "decimal", "float", "int", "integer", "boolean", "data", "time",
                                             "duration", "gYear", "gMonth",
                                             "dataTime"]:
            if type_["primitive_type"] is not None:
                if type_["primitive_type"].get("name", None) is not None:
                    merged_type["local_name"] = type_["primitive_type"]["name"]
                if type_["primitive_type"].get("local_name", None) is not None:
                    merged_type["local_name"] = type_["primitive_type"]["local_name"]

            elif type_["base_type"] is not None:
                if type_["base_type"].get("name", None) is not None:
                    merged_type["local_name"] = type_["base_type"]["name"]
                if type_["base_type"].get("local_name", None) is not None:
                    merged_type["local_name"] = type_["base_type"]["local_name"]
            elif type_["primitive_type"] is not None:
                if type_["primitive_type"].get("name", None) is not None:
                    merged_type["local_name"] = type_["primitive_type"]["name"]
                if type_["primitive_type"].get("local_name", None) is not None:
                    merged_type["local_name"] = type_["primitive_type"]["local_name"]
            elif type_["member_types"] is not None:
                if type_["member_types"].get("name", None) is not None:
                    merged_type["local_name"] = type_["member_types"]["name"]
                if type_["member_types"].get("local_name", None) is not None:
                    merged_type["local_name"] = type_["member_types"]["local_name"]
        if merged_type['name'] is None:
            merged_type['name'] = "string"
        if merged_type['local_name'] is None:
            merged_type['name'] = "string"
        return merged_type

    @staticmethod
    def add_keys(merged_type, type_, name):
        if type_[name] is not None:
            for key in type_[name].keys():
                if key not in merged_type.keys():
                    merged_type[key] = type_[name][key]

    @staticmethod
    def get_facets(node_name, node_type):
        type_ = None
        base_type = getattr(node_type, node_name, None)
        if base_type is not None:
            if isinstance(base_type, list):
                types = dict()
                for i, item in enumerate(base_type):
                    t = Fakers.all_facets_fot_type(f"member {i}", item)
                    if i > 0:
                        for key in t.keys():
                            if t[key] is not None and types.get(key, None) is None:
                                types[key] = t[key]
                    else:
                        types = t
                if types != dict():
                    type_ = types
                logger.trace(f"")
            else:
                type_ = Fakers.all_facets_fot_type(node_name, base_type)
        return type_

    @staticmethod
    def all_facets_fot_type(node_name, node_type):
        type_ = dict()
        logger.trace(f" node_name {node_name}   node_type: {node_type}")
        for attr in node_type.facets:
            if attr is not None:
                attr_name = attr.split("}")[1]
                type_[attr_name] = getattr(node_type.facets[attr], "value", None)
        name_attr = "name"
        name = getattr(node_type, name_attr, None)
        if name is not None:
            nn = name.split("}")
            type_[name_attr] = name.split("}")[1] if len(nn) > 1 else nn[0]
        else:
            type_[name_attr] = None
        type_["local_name"] = getattr(node_type, "local_name", None)
        type_["max_length"] = getattr(node_type, "max_length", None)
        type_["min_length"] = getattr(node_type, "min_length", None)
        type_["max_value"] = getattr(node_type, "max_value", None)
        type_["min_value"] = getattr(node_type, "min_value", None)
        type_["pattern"] = getattr(node_type, "pattern", None)
        patterns = getattr(node_type, "patterns", None)
        type_["patterns"] = getattr(patterns, "regexps", None) if patterns is not None else None

        type_["enumeration"] = getattr(node_type, "enumeration", None)

        return type_