import os
import traceback
import numpy as np
import numpy.random as npr
import tensorflow as tf
import keras.backend as K
from keras.models import load_model
from keras.layers import Conv2D, TimeDistributed, Flatten, Dense
from keras.models import Input, Model, Layer
from keras.applications import InceptionResNetV2
from keras.preprocessing.image import load_img, img_to_array
from utils import generate_anchors, draw_anchors, bbox_overlaps, bbox_transform,\
                    loss_cls,  parse_label, unmap, filter_boxes, \
                    clip_boxes, py_cpu_nms, bbox_transform_inv

# rpn_model = load_model('weights_3x3.hdf5',
#             custom_objects={'loss_cls': loss_cls,'smoothL1':smoothL1})
# x=rpn_model.predict(np.load('n02676566_6914')['fc'])
# rpn_model2 = load_model('weights_3x3.hdf5',
#             custom_objects={'loss_cls': loss_cls,'smoothL1':smoothL1})
# x=rpn_model2.predict(np.load('n02676566_6914')['fc'])
# print(x[1].shape)

##################  R-CNN Model  #######################
# RoI Pooling layer
class RoIPooling(Layer):
    def __init__(self, size=(7, 7)):
        self.size = size
        super(RoIPooling, self).__init__()

    def build(self, input_shape):
        self.shape = input_shape
        super(RoIPooling, self).build(input_shape)

    def call(self, inputs, **kwargs):
        ind=K.reshape(inputs[2],(-1,))
        x = K.tf.image.crop_and_resize(inputs[0], inputs[1], ind, self.size)
        return x

    def compute_output_shape(self, input_shape):
        a=input_shape[1][0]
        b=self.size[0]
        c=self.size[1]
        d=input_shape[0][3]
        return (a,b,c,d)

BATCH=128

feature_map=Input(batch_shape=(None,None,None,1536))
rois=Input(batch_shape=(None, 4))
ind=Input(batch_shape=(None, 1),dtype='int32')

p1=RoIPooling()([feature_map, rois, ind])

flat1 = Flatten()(p1)

fc1 = Dense(
        units=128,
        activation="relu",
        name="fc2"
    )(flat1)

output_deltas = Dense(
        units=4 * 200,
        activation="linear",
        kernel_initializer="zero",
        name="deltas2"
    )(fc1)

output_scores = Dense(
        units=1 * 200,
        activation="softmax",
        kernel_initializer="zero",
        name="scores2"
    )(fc1)

graph1 = tf.Graph()
with graph1.as_default():
    session1 = tf.Session()
    with session1.as_default():
        def smoothL1(y_true, y_pred):
            nd=K.tf.where(K.tf.not_equal(y_true,0))
            y_true=K.tf.gather_nd(y_true,nd)
            y_pred=K.tf.gather_nd(y_pred,nd)
            x = K.tf.losses.huber_loss(y_true,y_pred)
        #     x   = K.switch(x < HUBER_DELTA, 0.5 * x ** 2, HUBER_DELTA * (x - 0.5 * HUBER_DELTA))
            return x*4
        model=Model(inputs=[feature_map, rois, ind],outputs=[output_scores,output_deltas])
        model.summary()
        model.compile(optimizer='rmsprop',
                    loss={'deltas2':'mse', 'scores2':'categorical_crossentropy'})

##################  prepare batch  #######################
from wrap import CNN
rpn_model=CNN()
FG_FRAC=.25
FG_THRESH=.5
BG_THRESH_HI=.5
BG_THRESH_LO=.1
anchors_num = 9
small_box_threshold = 40

def produce_batch(feature_map, gt_boxes, h_w=None, category=None):
    height = np.shape(feature_map)[1]
    width = np.shape(feature_map)[2]
    num_feature_map=width*height

    w_stride = h_w[1] / width
    h_stride = h_w[0] / height
    #base anchors are 9 anchors wrt a tile (0,0,w_stride-1,h_stride-1)
    base_anchors=generate_anchors(w_stride,h_stride)
    shift_x = np.arange(0, width) * w_stride
    shift_y = np.arange(0, height) * h_stride
    shift_x, shift_y = np.meshgrid(shift_x, shift_y)
    shifts = np.vstack((shift_x.ravel(), shift_y.ravel(), shift_x.ravel(),
                            shift_y.ravel())).transpose()

    all_anchors = (base_anchors.reshape((1, anchors_num, 4)) +
                    shifts.reshape((1, num_feature_map, 4)).transpose((1, 0, 2)))
    total_anchors = num_feature_map * anchors_num
    all_anchors = all_anchors.reshape((total_anchors, 4))
    # 用训练好的rpn进行预测，得出scores和deltas
    res=rpn_model.query_cnn(feature_map)
    scores=res[0]
    scores=scores.reshape(-1,1)
    deltas=res[1]
    deltas=np.reshape(deltas,(-1,4))
    # 把dx dy转换成具体的xy值，并把照片以外的anchors去掉
    proposals =bbox_transform_inv(all_anchors, deltas)
    proposals = clip_boxes(proposals, (h_w[0],h_w[1]))
    # remove small boxes
    keep = filter_boxes(proposals, small_box_threshold) # here threshold is 40 pixel
    proposals = proposals[keep, :]
    scores = scores[keep]

    # sort socres and only keep top 6000.
    pre_nms_topN=6000
    order = scores.ravel().argsort()[::-1]
    if pre_nms_topN > 0:
        order = order[:pre_nms_topN]
    proposals = proposals[order, :]
    scores = scores[order]
    # apply NMS to to 6000, and then keep top 300
    post_nms_topN=300
    keep = py_cpu_nms(np.hstack((proposals, scores)), 0.7)
    if post_nms_topN > 0:
        keep = keep[:post_nms_topN]
    proposals = proposals[keep, :]
    scores = scores[keep]
    # 把ground true也加到proposals中
    proposals=np.vstack( (proposals, gt_boxes) )
    # calculate overlaps of proposal and gt_boxes
    overlaps = bbox_overlaps(proposals, gt_boxes)
    gt_assignment = overlaps.argmax(axis=1)
    max_overlaps = overlaps.max(axis=1)
    # labels = gt_labels[gt_assignment] #?

    # sub sample
    fg_inds = np.where(max_overlaps >= FG_THRESH)[0]
    fg_rois_per_this_image = min(int(BATCH*FG_FRAC), fg_inds.size)
    # Sample foreground regions without replacement
    if fg_inds.size > 0:
        fg_inds = npr.choice(fg_inds, size=fg_rois_per_this_image, replace=False)
    bg_inds = np.where((max_overlaps < BG_THRESH_HI) &
                       (max_overlaps >= BG_THRESH_LO))[0]
    bg_rois_per_this_image = BATCH - fg_rois_per_this_image
    bg_rois_per_this_image = min(bg_rois_per_this_image, bg_inds.size)
    # Sample background regions without replacement
    if bg_inds.size > 0:
        bg_inds = npr.choice(bg_inds, size=bg_rois_per_this_image, replace=False)
    # The indices that we're selecting (both fg and bg)
    keep_inds = np.append(fg_inds, bg_inds)
    # Select sampled values from various arrays:
    # labels = labels[keep_inds]
    rois = proposals[keep_inds]
    gt_rois=gt_boxes[gt_assignment[keep_inds]]
    targets = bbox_transform(rois, gt_rois)#input rois
    rois_num=targets.shape[0]
    batch_box=np.zeros((rois_num, 200, 4))
    for i in range(rois_num):
        batch_box[i, category] = targets[i]
    batch_box = np.reshape(batch_box, (rois_num, -1))
    # get gt category
    batch_categories = np.zeros((rois_num, 200, 1))
    for i in range(rois_num):
        batch_categories[i, category] = 1
    batch_categories = np.reshape(batch_categories, (rois_num, -1))
    return rois, batch_box, batch_categories

wider_face_dataset_path = "E:/Document/Datasets/Wider Face"
img_path = os.path.join(wider_face_dataset_path, 'WIDER_train')
gt_boxes_path =  os.path.join(wider_face_dataset_path, 'wider_face_split/wider_face_train_bbx_gt.txt')
feature_map_path='/home/jk/faster_rcnn/feature_maps/'
import glob
from multiprocessing import Process, Queue

# BATCH_SIZE=2048
def worker(path, q):
    print('worker start ' + path)
    batch_rois=[]
    batch_featuremap_inds=[]
    batch_categories=[]
    batch_bboxes=[]
    fc_index=0
    #'/ImageSets/DET/train_*'
    for fname in glob.glob(ILSVRC_dataset_path+path):
        print(fname)
        with open(fname,'r') as f:
            basename = os.path.basename(fname)
            category = int(basename.split('_')[1].split('.')[0])
            for line in f:
                if 'extra' not in line:
                    feature_map_file=feature_map_path+line.split()[0]
                    if not os.path.exists(feature_map_file):
                        continue

                    try:
                        feature_map=np.load(feature_map_file)['fc']
                        _, gt_boxes, h_w = parse_label(anno_path+line.split()[0]+'.xml')
                        if len(gt_boxes)==0:
                            continue
                        rois, bboxes, categories = produce_batch(feature_map, gt_boxes, h_w,category)
                    except Exception:
                        print('parse label or produce batch failed: for: '+line.split()[0])
                        traceback.print_exc()
                        continue
                    if len(rois) <= 0 :
                        continue

                    for i in range(len(rois)):
                        batch_rois.append(rois[i])
                        batch_featuremap_inds.append(fc_index)
                        batch_categories.append(categories[i])
                        batch_bboxes.append(bboxes[i])
                    a=feature_map
                    b=np.asarray(batch_rois)
                    c=np.asarray(batch_featuremap_inds)
                    d=np.asarray(batch_categories)
                    e=np.asarray(batch_bboxes)
                    f=np.zeros((len(rois),a.shape[1],a.shape[2],a.shape[3]))
                    f[0]=feature_map[0]
                    q.put([a,b,c,d,e])
                    # yield [f,b,c], [d,e]
                    batch_rois=[]
                    batch_featuremap_inds=[]
                    batch_categories=[]
                    batch_bboxes=[]
                    fc_index=0

def worker(gt_data, q, batch_size=256):
    batch_rois=[]
    batch_featuremap_inds=[]
    batch_categories=[]
    batch_bboxes=[]
    fc_index=0
    for path, gt_boxes in gt_data.items():
        path = path[:-4] #村的时候就没有带.jpg
        feature_map=np.load(path)['fc']
        tiles, labels, bboxes = produce_batch(feature_map, gt_boxes)
        if tiles is None or labels is None or bboxes is None:
            print("continue")
            continue
        if len(rois) <= 0 :
                continue
        for i in range(len(rois)):
            batch_rois.append(rois[i])
            batch_featuremap_inds.append(fc_index)
            batch_categories.append(categories[i])
            batch_bboxes.append(bboxes[i])
        a=feature_map
        b=np.asarray(batch_rois)
        c=np.asarray(batch_featuremap_inds)
        d=np.asarray(batch_categories)
        e=np.asarray(batch_bboxes)
        f=np.zeros((len(rois),a.shape[1],a.shape[2],a.shape[3]))
        f[0]=feature_map[0]
        if not a.any() or not b.any() or not c.any(): #if a或b或c中所有的的值都是0
            print("empty array found.") # 当gt_box比较小的时候就会这样
            if not a.any():
                print("It is because a.any() is False")
            if not b.any():
                print("It is because b.any() is False")
            if not c.any():
                print("It is because c.any() is False", c)

        q.put([a,b,c,d,e])
        # yield [f,b,c], [d,e]
        batch_rois=[]
        batch_featuremap_inds=[]
        batch_categories=[]
        batch_bboxes=[]
        fc_index=0

q = Queue(20)

p1 = Process(target=worker, args=('/ImageSets/DET/train_*[0-1].txt',q))
p1.start()
p2 = Process(target=worker, args=('/ImageSets/DET/train_*[2-3].txt',q))
p2.start()
# p3 = Process(target=worker, args=('/ImageSets/DET/train_*[4-6].txt',q))
# p3.start()
# p4 = Process(target=worker, args=('/ImageSets/DET/train_*[7-9].txt',q))
# p4.start()

def input_generator():
    while 1:
        batch = q.get()
        yield [batch[0], batch[1], batch[2]], [batch[3], batch[4]]

# model.load_weights('./weights.hdf5')
from keras.callbacks import ModelCheckpoint
checkpointer = ModelCheckpoint(filepath='./rcnn_weights_1.hdf5', monitor='loss', verbose=1, save_best_only=True)
with session1.as_default():
    model.fit_generator(input_generator(), steps_per_epoch=3000, epochs=100, callbacks=[checkpointer])


"""Embedding worker."""
from worker import app
from docker_logs import get_logger
from pymagnitude import Magnitude
import gensim
import numpy as np
from mongo_task import mongo_task

logging = get_logger("embedding_task")
logging.propagate = False

def preproces_text(text):
    """Text splitted to tokens."""
    return gensim.utils.simple_preprocess(text)

def get_sentences_representation(vectors, splitted_sentence):
    """Counts average embedding."""
    length = 0
    av_sum = np.zeros(shape=(100,))
    for i in range(len(splitted_sentence)):
        if splitted_sentence[i] in vectors:
            av_sum = av_sum + vectors.query(splitted_sentence[i])
            length += 1
    if length > 0:
        av_sum = av_sum / length
    return av_sum

def get_text_embedding(vectors, text):
    """Gets text embedding."""
    return get_sentences_representation(vectors, preproces_text(text)).tolist()

@app.task(bind=True, name='embedding_task', queue='embedding')
def embedding_task(self, tweets):
    """Embedding task."""
    logging.info(f"DATAFRAME PASSED FOR EMBEDDING SIZE {len(tweets)} ")
    vectors = Magnitude("glove_vectors/glove.6B.100d.magnitude")
    for tweet in tweets:
        tweet.embedding = get_text_embedding(vectors, tweet.text)
    mongo_task(tweets)
    return tweets


'''Tests for graphite-opentsdb.'''
# pylint: disable=C0103
# pylint: disable=R0904

from django import test
from django.core.cache import cache
from httmock import all_requests, with_httmock, HTTMock
import mock

from graphite_opentsdb.finder import OpenTSDBFinder
from graphite.storage import FindQuery
from graphite_opentsdb import app_settings

@all_requests
def mocked_urls(url, request):
    return {
        ('localhost:4242', '/api/v1/tree/branch', 'branch=0001'): {
            'status_code': 200,
            'content': '''
                {
                    "branchId": "0001",
                    "branches": [
                        {
                            "branchId": "0001CFE0B4A4",
                            "branches": null,
                            "depth": 1,
                            "displayName": "branch1",
                            "leaves": null,
                            "path": {
                                "0": "ROOT",
                                "1": "branch1"
                            },
                            "treeId": 1
                        },
                        {
                            "branchId": "00013FFD49C8",
                            "branches": null,
                            "depth": 1,
                            "displayName": "branch2",
                            "leaves": null,
                            "path": {
                                "0": "ROOT",
                                "1": "branch2"
                            },
                            "treeId": 1
                        }
                    ],
                    "depth": 0,
                    "displayName": "ROOT",
                    "leaves": [
                        {
                            "displayName": "leaf",
                            "metric": "leaf",
                            "tags": {
                                "host": "localhost"
                            },
                            "tsuid": "000BC700000100047A"
                        }
                    ],
                    "path": {
                        "0": "ROOT"
                    },
                    "treeId": 1
                }
            ''',
        },
        ('localhost:4242', '/api/v1/tree/branch', 'branch=0001CFE0B4A4'): {
            'status_code': 200,
            'content': '''
                {
                    "branchId": "0001CFE0B4A4",
                    "branches": null,
                    "depth": 1,
                    "displayName": "branch1",
                    "leaves": [
                        {
                            "displayName": "leaf",
                            "metric": "branch1.leaf",
                            "tags": {
                                "host": "localhost"
                            },
                            "tsuid": "000BC700000100047B"
                        }
                    ],
                    "path": {
                        "0": "ROOT",
                        "1": "branch1"
                    },
                    "treeId": 1
                }
            ''',
        },
        ('localhost:4242', '/api/v1/tree/branch', 'branch=00013FFD49C8'): {
            'status_code': 200,
            'content': '''
                {
                    "branchId": "00013FFD49C8",
                    "branches": null,
                    "depth": 1,
                    "displayName": "branch2",
                    "leaves": [
                        {
                            "displayName": "leaf",
                            "metric": "branch2.leaf",
                            "tags": {
                                "host": "localhost"
                            },
                            "tsuid": "000BC700000100047C"
                        }
                    ],
                    "path": {
                        "0": "ROOT",
                        "1": "branch2"
                    },
                    "treeId": 1
                }
            ''',
        },
        ('localhost:4242', '/api/v1/tree/branch', 'branch=0002'): {
            'status_code': 200,
            'content': '''
                {
                    "branchId": "0002",
                    "branches": [
                        {
                            "branchId": "0002CFE0B4A4",
                            "branches": null,
                            "depth": 1,
                            "displayName": "branch.with.dots",
                            "leaves": null,
                            "path": {
                                "0": "ROOT",
                                "1": "branch.with.dots"
                            },
                            "treeId": 1
                        }
                    ],
                    "depth": 0,
                    "displayName": "ROOT",
                    "leaves": null,
                    "path": {
                        "0": "ROOT"
                    },
                    "treeId": 2
                }
            ''',
        },
        ('localhost:4242', '/api/v1/tree/branch', 'branch=0002CFE0B4A4'): {
            'status_code': 200,
            'content': '''
                {
                    "branchId": "0002CFE0B4A4",
                    "branches": null,
                    "depth": 1,
                    "displayName": "branch.with.dots",
                    "leaves": [
                        {
                            "displayName": "leaf.with.dots",
                            "metric": "branch.with.dots.leaf.with.dots",
                            "tags": {
                                "host": "localhost"
                            },
                            "tsuid": "000BC700000100047D"
                        }
                    ],
                    "path": {
                        "0": "ROOT",
                        "1": "branch.with.dots"
                    },
                    "treeId": 2
                }
            ''',
        },
        ('localhost:4242', '/api/v1/tree/branch', 'branch=0003'): {
            'status_code': 404,
            'content': '',
        },
    }.get(
        (url.netloc, url.path, url.query),
        {
            'status_code': 500,
            'content': '',
        }
    )

@all_requests
def bad_urls(url, request):
    return {
        'status_code': 500,
        'content': '',
    }

class OpenTSDBFinderTestCase(test.TestCase):
    '''Test the finder class.'''

    def setUp(self):
        #self.settings_dict = copy.deepcopy(self.BASE_SETTINGS)
        self.finder = OpenTSDBFinder('http://localhost:4242/api/v1', 1)
        cache.clear()

    @mock.patch.object(app_settings, 'OPENTSDB_URI', 'http://localhost:9999')
    @mock.patch.object(app_settings, 'OPENTSDB_TREE', 999)
    def test_finder_settings(self):
        '''
        Test that the finder can default to Django settings.
        '''

        finder = OpenTSDBFinder()
        self.assertEqual(finder.opentsdb_uri, 'http://localhost:9999')
        self.assertEqual(finder.opentsdb_tree, 999)

    @with_httmock(mocked_urls)
    def test_finder_nodes(self):
        '''
        Test that the finder can find nodes.
        '''

        # Base query
        nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None)))

        self.assertEqual(
            [node.path for node in nodes],
            ['branch1', 'branch2', 'leaf'],
        )

        # One level down
        nodes = list(self.finder.find_nodes(query=FindQuery('*.leaf', None, None)))

        self.assertEqual(
            [node.path for node in nodes],
            ['branch1.leaf', 'branch2.leaf']
        )

        nodes = list(self.finder.find_nodes(query=FindQuery('branch1.leaf', None, None)))

        self.assertEqual(
            [node.path for node in nodes],
            ['branch1.leaf'],
        )

    @with_httmock(mocked_urls)
    def test_finder_braces(self):
        '''
        Test that the finder can deal with brace expressions.
        '''

        nodes = list(self.finder.find_nodes(query=FindQuery('{branch1,leaf}', None, None)))
        self.assertEqual(
            [node.path for node in nodes],
            ['branch1', 'leaf'],
        )

    @with_httmock(mocked_urls)
    def test_finder_character_classes(self):
        '''
        Test that the finder can deal with character classes.
        '''

        nodes = list(self.finder.find_nodes(query=FindQuery('branch[1]', None, None)))
        self.assertEqual(
            [node.path for node in nodes],
            ['branch1'],
        )

    @with_httmock(mocked_urls)
    def test_finder_dotted_nodes(self):
        '''
        Test that the finder can deal with nodes with dots
        (Since OpenTSDB allows that).
        '''

        finder = OpenTSDBFinder('http://localhost:4242/api/v1/', 2)

        nodes = list(finder.find_nodes(query=FindQuery('*', None, None)))
        self.assertEqual(
            [node.name for node in nodes],
            ['branch.with.dots'],
        )

    @with_httmock(mocked_urls)
    def test_finder_nested_dotted_nodes(self):
        '''
        Test that the finder can deal with nested nodes with dots
        (Since OpenTSDB allows that).
        '''

        finder = OpenTSDBFinder('http://localhost:4242/api/v1/', 2)

        nodes = list(finder.find_nodes(query=FindQuery('*.*', None, None)))
        self.assertEqual(
            [node.name for node in nodes],
            ['leaf.with.dots'],
        )

    @with_httmock(mocked_urls)
    def test_finder_missing_branch(self):
        '''
        Test that the finder can deal with a missing branch.
        '''

        finder = OpenTSDBFinder('http://localhost:4242/api/v1/', 3)

        with self.assertRaises(ValueError):
            nodes = list(finder.find_nodes(query=FindQuery('*', None, None)))

    def test_caching(self):
        '''
        Test caching behaviour.
        '''
        with HTTMock(mocked_urls):
            nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None)))

            self.assertEqual(
                [node.path for node in nodes],
                ['branch1', 'branch2', 'leaf'],
            )

        with HTTMock(bad_urls):
            nodes = list(self.finder.find_nodes(query=FindQuery('*', None, None)))

            self.assertEqual(
                [node.path for node in nodes],
                ['branch1', 'branch2', 'leaf'],
            )


import json

from django import forms
from django.forms import BooleanField, CharField, ChoiceField, Field, FileField, Form, ValidationError
from django.utils.encoding import smart_text
from django.utils.translation import ugettext as _
from django.utils.translation import ugettext_lazy as ugettext

def html_escape(s):
    """Replace special characters "&", "<" and ">" to HTML-safe sequences.
    If the optional flag quote is true, the quotation mark character (")
    is also translated.
    """
    s = s.replace("&", "&")
    s = s.replace("<", "<")
    s = s.replace(">", ">")
    s = s.replace(" ", " ")
    s = s.replace('"', """)
    s = s.replace("'", "'")

    return s

def html_exempt(s):
    s = s.replace("&", "&")
    s = s.replace("<", "<")
    s = s.replace(">", ">")
    s = s.replace(" ", " ")
    s = s.replace(""", '"')
    s = s.replace("'", "'")

    return s

class BaseForm(Form):
    def __init__(self, *args, **kwargs):
        if "is_create" in kwargs:
            if not kwargs["is_create"] and "table_name" in self.base_fields:
                required = self.base_fields["table_name"].required
                self.base_fields["table_name"] = forms.RegexField(
                    required=required,
                    regex=r"^[a-zA-Z][_a-zA-Z0-9]*$",
                    error_messages={"invalid": _("由英文字母、下划线或数字组成，且以字母开头")},
                    max_length=255,
                    label=_("计算标识"),
                )
            del kwargs["is_create"]
        super(BaseForm, self).__init__(*args, **kwargs)

    # strip leading or trailing whitespace
    def _clean_fields(self):
        for name, field in list(self.fields.items()):
            # value_from_datadict() gets the data from the data dictionaries.
            # Each widget type knows how to retrieve its own data, because some
            # widgets split data over several HTML fields.
            value = field.widget.value_from_datadict(self.data, self.files, self.add_prefix(name))
            try:
                if isinstance(field, FileField):
                    initial = self.initial.get(name, field.initial)
                    value = field.clean(value, initial)
                else:
                    if isinstance(value, str):
                        value = field.clean(value.strip())
                    else:
                        value = field.clean(value)
                self.cleaned_data[name] = value
                if hasattr(self, "clean_%s" % name):
                    value = getattr(self, "clean_%s" % name)()
                    self.cleaned_data[name] = value
            except ValidationError as e:
                self.add_error(name, e)

    def format_errmsg(self):
        errors = self.errors
        # django 1.8，取declared_fields
        declared_fields = self.declared_fields
        errmsg_arr = []
        for _key in errors:
            _keymsg_arr = errors[_key]
            if _key in declared_fields:
                _label = declared_fields[_key].label
                label = _label if _label else _key
                _errmsg = []
                for _err in _keymsg_arr.data:
                    if _err.code in ["max_length", "min_length"]:
                        # 为了兼容django1.8 zh-hans表单中文翻译bug的问题
                        _msg = ugettext(_err.message % _err.params.get("limit_value", 0)) % _err.params
                    else:
                        _msg = _err.message
                    _errmsg.append("{}，{}".format(label, _msg))

                errmsg_arr.extend(_errmsg)
            if _key == "__all__":
                errmsg_arr.extend(_keymsg_arr)
        return errmsg_arr[0]

class MultiStrSplitByCommaField(Field):
    """
    多个字段，使用逗号隔开
    """

    def to_python(self, value):
        if not value:
            return []

        return [_v for _v in value.split(",") if _v != ""]

    def validate(self, value):
        super(MultiStrSplitByCommaField, self).validate(value)

class MultiStrSplitBySemiconField(Field):
    """
    多个字段，使用分号隔开
    """

    def to_python(self, value):
        if not value:
            return []

        return [_v for _v in value.split(";") if _v != ""]

    def validate(self, value):
        super(MultiStrSplitBySemiconField, self).validate(value)

class MulChoiceFieldByComma(ChoiceField):
    """
    多选字段，使用逗号隔开
    """

    default_error_messages = {
        "invalid_choice": _("Select a valid choice. %(value)s is not one of the available choices."),
        "invalid_list": _("Enter a list of values."),
    }

    def to_python(self, value):
        if not value:
            return []
        else:
            value = value.split(",")
        return [smart_text(val) for val in value]

    def validate(self, value):
        """
        Validates that the input is a list or tuple.
        """
        if self.required and not value:
            raise ValidationError(self.error_messages["required"])
        # Validate that each value in the value list is in self.choices.
        for val in value:
            if not self.valid_value(val):
                raise ValidationError(self.error_messages["invalid_choice"] % {"value": val})

class ListField(Field):
    """
    列表字段
    """

    def to_python(self, value):
        if value is None:
            return []

        if not (isinstance(value, list)):
            raise ValidationError(_("字段类型必须为数组"))
        return value

    def validate(self, value):
        super(ListField, self).validate(value)

class DictField(Field):
    """
    字典字段
    """

    def to_python(self, value):
        if value is None:
            return None
        if not (isinstance(value, dict)):
            raise ValidationError(_("字段类型必须为字典类型"))
        return value

    def validate(self, value):
        super(DictField, self).validate(value)

class JSONField(Field):
    def to_python(self, value):
        if value is None:
            return None
        try:
            value = json.loads(value)
        except (TypeError, ValueError):
            try:
                # 若本身就是json格式，直接返回，为了兼容自更新情况
                json.dumps(value)
            except (TypeError, ValueError):
                raise ValidationError(_("字段格式不对，必须符合json格式"))
            else:
                return value
        return value

class HtmlEscapeCharField(CharField):
    """
    转义 html 相关字符，其他使用方式与 CharField 一致
    """

    def to_python(self, value):
        raw_value = super(HtmlEscapeCharField, self).to_python(value)
        return html_exempt(raw_value)

    def clean(self, value):
        cleaned_value = super(HtmlEscapeCharField, self).clean(value)
        return html_escape(cleaned_value)

def check_ip_rule(ipaddr):
    ipaddr = str(ipaddr)
    addr = ipaddr.strip().split(".")  # 切割IP地址为一个列表
    # print addr
    if len(addr) != 4:  # 切割后列表必须有4个参数
        return False
    for i in range(4):
        try:
            addr[i] = int(addr[i])  # 每个参数必须为数字，否则校验失败
        except BaseException:
            return False
        if addr[i] <= 255 and addr[i] >= 0:  # 每个参数值必须在0-255之间
            pass
        else:
            return False
        i += 1
    else:
        return True

class BooleanFieldsRequiredMixin(Form):
    """
    当BooleanField通过设置required为True时要求用户必须传入该参数，但允许该字段值为False，原表单认证会不通过
    为解决这种情况，表单类继承该类，嗯，就是这么坑
    1. https://stackoverflow.com/questions/10440937/what-is-wrong-with-the-django-forms-booleanfield-unit-test-case
    2. https://code.djangoproject.com/ticket/5957
    """

    def clean(self):
        for field_name, field in list(self.fields.items()):
            # Only BooleanField not subclasses of it.
            if not isinstance(field, BooleanField):
                continue

            if field_name not in self.data:
                self._errors[field_name] = self.error_class([ValidationError(_("该参数为必需项"))])

        return super(BooleanFieldsRequiredMixin, self).clean()

class CheckboxInput(forms.CheckboxInput):
    """
    为 BooleanField 设置默认值共用类
    """

    def __init__(self, default=False, *args, **kwargs):
        super(CheckboxInput, self).__init__(*args, **kwargs)
        self.default = default

    def value_from_datadict(self, data, files, name):
        if name not in data:
            return self.default
        return super(CheckboxInput, self).value_from_datadict(data, files, name)


import os
import sys
import torch
from torch.utils.data import DataLoader
from torch import nn, optim
from torchvision.models import resnet18
from utils import Flatten
from DataLoaders import DataLoaders

batchsz = 32
lr = 1e-3
epochs = 10
device = torch.device("cuda")
torch.manual_seed(1234)
abs_path = os.getcwd()
abs_path = abs_path+'/'
# model_name ="mdl/"+ str(sys.argv[3])
# dataset_name ="dataset/"+ str(sys.argv[2])
# train_db = DataLoaders(dataset_name, 224, mode="train")
# val_db = DataLoaders(dataset_name, 224, mode="val")
# test_db = DataLoaders(dataset_name, 224, mode="test")
#
# train_loader = DataLoader(train_db, batch_size=batchsz, shuffle=True, num_workers=2)
# test_loader = DataLoader(test_db, batch_size=batchsz, num_workers=1)
# val_loader = DataLoader(val_db, batch_size=batchsz, num_workers=2)
model_name =abs_path+"mdl/"
dataset_name =abs_path+"dataset/"
train_db =""
val_db =""
test_db=""

train_loader = ""
test_loader = ""
val_loader = ""

# viz = visdom.Visdom()
def evalute(model, loader):
    correct = 0
    total = len(loader.dataset)
    for x, y in loader:
        x, y = x.to(device), y.to(device)
        x.squeeze(0)

        with torch.no_grad():
            logits = model(x)
            pred = logits.argmax(dim=1)

        correct += torch.eq(pred, y).sum().float().item()
    return correct / total

def train(model, isFirst):
    if isFirst == 0:
        model.load_state_dict(torch.load(model_name))
    optimizer = optim.Adam(model.parameters(), lr=lr)
    criteon = nn.CrossEntropyLoss()
    best_acc, best_epoch = 0, 0
    for epoch in range(epochs):
        for step, (x, y) in enumerate(train_loader):
            x, y = x.to(device), y.to(device)
            logits = model.forward(x)
            loss = criteon(logits, y)
            optimizer.zero_grad()
            loss.backward()
            optimizer.step()
        if epoch % 1 == 0:
            val_acc = evalute(model, val_loader)
            if val_acc > best_acc:
                best_epoch = epoch
                best_acc = val_acc
                torch.save(model.state_dict(), model_name)
            print("epoch:",epoch, "   acc:",val_acc)
    print('best acc:', best_acc, 'best epoch:', best_epoch)

def test(model):
    model.load_state_dict(torch.load(model_name))

    print("loaded from skpt")
    test_acc = evalute(model, test_loader)
    print('test acc:', test_acc)

def load_list(root):
    name2label = {}
    for name in sorted(os.listdir(os.path.join(root))):
        if not os.path.isdir(os.path.join(root, name)):
            continue
        name2label[name] = len(name2label.keys())
    print(name2label)
    return name2label
def init():
    global model_name
    global dataset_name
    model_name = model_name+"pokeman.mdl"
    dataset_name = dataset_name+"pokeman"
    global train_db
    global val_db
    global test_db
    train_db = DataLoaders(dataset_name, 224, mode="train")
    val_db = DataLoaders(dataset_name, 224, mode="val")
    test_db = DataLoaders(dataset_name, 224, mode="test")

    global train_loader
    global test_loader
    global val_loader
    train_loader = DataLoader(train_db, batch_size=batchsz, shuffle=True, num_workers=2)
    test_loader = DataLoader(test_db, batch_size=batchsz, num_workers=1)
    val_loader = DataLoader(val_db, batch_size=batchsz, num_workers=2)

def main():

    init()
    # model = ResNet18(5).to(device)
    root =abs_path+ "dataset/pokeman"
    name2label = load_list(root)
    lens = len(name2label)
    isFirst = 1
    train_model = resnet18(pretrained=True)
    model = nn.Sequential(*list(train_model.children())[:-1],
                          Flatten(),
                          nn.Linear(512, lens),
                          ).to(device)
    isTrain = 1
    if isTrain == 1:
        train(model, isFirst)
    else:
        test(model)

if __name__ == '__main__':
    main()


import maya.cmds as cmds
import maya.mel
s = cmds.ls(selection = True)
axes = ['X','Y','Z']

for axis in axes:
    c=0
    while(c<=360):
        for a in s:
            x = a +"."+"rotate" +axis
            cmds.setAttr(x,c)

        mel.eval('renderWindowRender redoPreviousRender renderView')
        editor =  'renderView'
            #c= c+5
        cmds.renderWindowEditor( editor, e=True,refresh = True, writeImage=('/Users/tinyteddybear/Documents/Sword/Sword'+axis+str(c)))
        c=c+10


from ast import dump
from random import getstate
import re
from typing import final
from flask import Flask, render_template,request, url_for, flash, redirect,current_app
from flask_mysqldb import MySQL
from flask_wtf import FlaskForm
from wtforms import StringField, SubmitField
from wtforms.validators import DataRequired

class State(FlaskForm):
    statename = StringField(label=('Enter state Name:'), validators=[DataRequired()])
    stateab = StringField(label=('Enter state Ab:'), validators=[DataRequired()])
    submit = SubmitField(label=('Submit'))

app = Flask(__name__)
# Config MySQL
app.config['MYSQL_HOST'] = 'localhost'
app.config['MYSQL_USER'] = 'root'
app.config['MYSQL_PASSWORD'] = ''
app.config['MYSQL_DB'] = 'artgallery'
app.config['SECRET_KEY'] = 'df0331cefc6c2b9a5d0208a726a5d1c0fd37324feba25506'
mysql = MySQL(app)
def getCustomers(state):
    query = '''select c.cID, c.name, c.street, c.city, c.stateAb,s.stateName, c.zipcode
from customer c
join state s
on s.stateAb = c.stateAb
where s.stateAb like %s or s.stateName like %s '''
    cur = mysql.connection.cursor()
    cur.execute(query, (state,state))
    rv = cur.fetchall()
    print(cur._executed)
    cur.close()
    return rv

def getStates():
        query = '''select * from state'''
        cur = mysql.connection.cursor()
        cur.execute(query)
        rv = cur.fetchall()
        print(rv)
        flash(cur._executed)
        cur.close()
        return rv

def createState(statename,stateab):
    try:
        query = '''insert into state(statename,stateab) values(%s,%s)'''
        cur = mysql.connection.cursor()
        cur.execute(query,(statename,stateab))
        mysql.connection.commit()
        cur.close()
    except:
        print('exception')
        mysql.connection.rollback()
    # finally:
        # mysql.connection.close()

def updateState(oldstatename,statename,stateab):
    try:
        print(oldstatename)
        print(statename)
        print(stateab)
        query = '''update state set statename = %s, stateab= %s where statename = %s'''
        cur = mysql.connection.cursor()
        cur.execute(query,(statename,stateab,oldstatename))
        print(cur._executed)
        mysql.connection.commit()
        cur.close()
    except:
        print('exception')
        mysql.connection.rollback()

def removestate(statename):
    try:
        cur = mysql.connection.cursor()
        cur.execute('''delete from state where statename like %s''',(statename,))
        mysql.connection.commit()
        cur.close()
    except:
        flash('cannot delete foreign key ')
    # finally:
        # mysql.connection.close()

@app.route('/')
def home():
    return render_template('index.html')

@app.route('/arts', methods = ('GET', 'POST'))
def listArts():
    # data
     if request.method == 'GET':
        return render_template('arts.html',result=getCustomers('texas') )
     else:
        state = request.form['state']
        if not state:
            flash('state is required!')
        else:
            return render_template('arts.html',result=getCustomers(state))

@app.route('/add')
def addArt():
    return render_template('add.html')

@app.route('/states', methods = ('GET', 'POST'))
def states():
    form = State()
    if request.method == 'POST' and form.validate_on_submit():
        print('valid form')
        statename = request.form['statename']
        stateab = request.form['stateab']
        createState(statename,stateab)
        return redirect('/states')
    if request.method == 'GET':
        return render_template('states.html',result=getStates(),form=form)

@app.route('/deletestate/<id>')
def deletestate(id):
    removestate(id)
    return redirect('/states')

@app.route('/updatestate', methods = ['POST'])
def updatestate():
    form = State()
    if request.method == 'POST' and form.validate_on_submit():
        print('valid form')
        oldstatename = request.form['oldstatename']
        statename = request.form['statename']
        stateab = request.form['stateab']
        updateState(oldstatename,statename,stateab)
        return redirect('/states')


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

class Solution:
    def __init__(self) -> None:
        # TODO:
        # Load data from data/chipotle.tsv file using Pandas library and
        # assign the dataset to the 'chipo' variable.
        file = 'data/chipotle.tsv'
        self.chipo = pd.read_csv(file, sep='\t')

    def top_x(self, count) -> None:
        # TODO
        # Top x number of entries from the dataset and display as markdown format.
        topx = self.chipo.head(count)
        print(topx.to_markdown())

    def count(self) -> int:
        # TODO
        # The number of observations/entries in the dataset.
        return self.chipo.shape[0]

    def info(self) -> None:
        # TODO
        # print data info.
        return self.chipo.info()

    def num_column(self) -> int:
        # TODO return the number of columns in the dataset
        # print(self.chipo.shape[1])
        # for index in self.chipo.columns['index']
        #     print(index)
        return self.chipo.shape[1]

    def print_columns(self) -> None:
        # TODO Print the name of all the columns.
        # return self.chipo.columns.to_series()
        return self.chipo.columns.to_list()

    def most_ordered_item(self):
        # TODO
        a = self.chipo[['item_name','quantity','order_id']]
        # print(a)
        a = a.groupby('item_name',as_index=False)
        # print(a)
        a = a.sum()
        # print(a)
        a = a.sort_values(by=['quantity'],ascending = False)
        # print(a)
        a = a.head(1)
        # print(a)
        item_name = a['item_name'].values[0]
        order_id = a['order_id'].values[0]
        quantity = a['quantity'].values[0]
        return item_name, order_id, quantity

    def total_item_orders(self) -> int:
       # TODO How many items were orderd in total?
        return self.chipo['quantity'].sum()

    def total_sales(self) -> float:
        # TODO
        # 1. Create a lambda function to change all item prices to float.
        # 2. Calculate total sales.
        # df_list = self.chipo['item_price'].value_counts()
        # df_list2 = self.chipo['quantity'].to_list()
        # x = self.chipo
        # self.chipo['item_price'] = self.chipo['item_price'][1:-1]
        # print('***  ' + self.chipo['item_price'])
        # x= lambda a:a'

        a = self.chipo['item_price'].apply(lambda x:float(x.replace('$','')))
        aa = self.chipo['quantity'] * a
        # print(a)
        # pd.np.multiply(self.chipo['item_price'],)
        # sum = 0.0
        # for i in df_list:
        #     sum+=float(i.replace('$',''))
        # print(sum)
        return aa.sum()

    def num_orders(self) -> int:
        # TODO
        # How many orders were made in the dataset?
        # print(self.chipo['order_id'].tail(1))
        # print(self.chipo['order_id'].tail(1).iat[0])
        return self.chipo['order_id'].tail(1).iat[0]

    def average_sales_amount_per_order(self) -> float:
        # TODO
        # print(round(Solution.total_sales(self)/Solution.num_orders(self),2))
        return round(Solution.total_sales(self)/Solution.num_orders(self),2)

    def num_different_items_sold(self) -> int:
        # TODO
        # How many different items are sold?
        # self.chipo['item_name'].unique()
        # print(self.chipo['item_name'].unique().shape)
        return self.chipo['item_name'].unique().shape[0]

    def plot_histogram_top_x_popular_items(self, x: int) -> None:
        from collections import Counter
        letter_counter = Counter(self.chipo.item_name)
        # print(type(letter_counter))
        # print(letter_counter)
        # print(letter_counter.items())
        # print(pd.DataFrame(letter_counter.items()))
        # TODO
        # 1. convert the dictionary to a DataFrame
        # print(pd.DataFrame.from_dict(letter_counter))
        df = pd.DataFrame(letter_counter.items())
        # print(df)
        # 2. sort the values from the top to the least value and slice the first 5 items
        # print(df.sort_values(1, ascending=False).head(x))
        final = df.sort_values(1, ascending=False).head(x)
        # 3. create a 'bar' plot from the DataFrame
        final.plot.bar(x=0,y=1,rot=0)
        # 4. set the title and labels:
        #     x: Items
        #     y: Number of Orders
        #     title: Most popular items
        plt.xlabel('Items')
        plt.ylabel('Number of Orders')
        plt.suptitle('Most popular items')
        # 5. show the plot. Hint: plt.show(block=True).
        plt.show(block = True)
        # pass

    def scatter_plot_num_items_per_order_price(self) -> None:
        # TODO
        # 1. create a list of prices by removing dollar sign and trailing space.
        self.chipo['item_price'] = self.chipo['item_price'].apply(lambda x:float(x.replace('$','')))
        # print(self.chipo)
        # 2. groupby the orders and sum it.
        orderedChipo = self.chipo.groupby('order_id').sum()
        # print(orderedChipo)
        # a = self.chipo['item_price'].replace('$','')
        # print(a)

        # 3. create a scatter plot:
        #       x: orders' item price
        #       y: orders' quantity
        #       s: 50
        #       c: blue
        orderedChipo.plot.scatter(x='item_price',y='quantity',s=50,c='blue')
        plt.xlabel('Order Price')
        plt.ylabel('Num Items')
        plt.suptitle('Numer of items per order price')
        # 4. set the title and labels.
        #       title: Numer of items per order price
        #       x: Order Price
        #       y: Num Items
        plt.show(block = True)

def test() -> None:
    solution = Solution()
    solution.top_x(10)
    count = solution.count()
    print(count)
    assert count == 4622
    solution.info()
    print(solution.print_columns())
    count = solution.num_column()
    assert count == 5
    item_name, order_id, quantity = solution.most_ordered_item()
    assert item_name == 'Chicken Bowl'
    assert order_id == 713926
    assert quantity == 761
    total = solution.total_item_orders()
    assert total == 4972
    assert 39237.02 == solution.total_sales()
    assert 1834 == solution.num_orders()
    assert 21.39 == solution.average_sales_amount_per_order()
    assert 50 == solution.num_different_items_sold()
    solution.plot_histogram_top_x_popular_items(5)
    solution.scatter_plot_num_items_per_order_price()

if __name__ == "__main__":
    # execute only if run as a script
    test()


import logging
from django.contrib.auth import get_user_model
from django.db.models import Q
from account import models
from account.models import GroupSubcategory, Category, SubCategory

User = get_user_model()

logging.basicConfig(format=u'%(levelname)-8s [%(asctime)s] %(message)s', level=logging.DEBUG,
                    filename=u'erp.log')

def get_user_info(user_data):
    user = user_data.get("user")
    user_obj = User.objects.get(id=user['id'])

    user_data["roles"] = list(user_obj.groups.values('id', 'name'))

    categorys_data = get_user_categorys_data2(user_obj)

    return {
        "categories": categorys_data,
        "user": user_data
    }

def get_user_accept_subcategorys(user):
    subcategorys_set = set()
    for group in user.groups.all():
        if GroupSubcategory.objects.filter(group=group).exists():
            subcategorys = set(GroupSubcategory.objects.get(group=group).links.exclude(status=3).order_by('priority'))
            subcategorys_set = subcategorys_set.union(subcategorys)

    return subcategorys_set

def get_user_categorys_data(user):
    temp = {}

    subcategorys = get_user_accept_subcategorys(user)

    for subcategory in subcategorys:
        category = subcategory.get_category()
        if category:
            if not temp.get(category.name):
                temp[category.name] = category.get_data()
                temp[category.name]["links"] = list()

            temp[category.name]["links"].append(subcategory.get_data())

    result = sorted([temp[key] for key in temp], key=lambda k: k['priority'])
    if result:
        return result
    return None

def get_subcategory(link):
    sub_cat = SubCategory.objects.filter(link=link)

    if sub_cat.exists():
        return sub_cat[0]

    return None


import os
import argparse
import re

def delDirVirus(fol):
    for root, dirs, files in os.walk(fol):
        filename = str(root).split('/')[-1]
        pattern = fr'^({filename})+(\s|)+(.exe)$'
        for file in filter(lambda x: re.match(pattern,x), files):
            os.remove(os.path.join(root,file))

def delUnhideVirus(fol):
    for root, dirs, files in os.walk(fol):
        filename = str(root).split('/')[-1]
        filename = re.escape(filename)
        pattern = fr'^({filename})+(\s|)+(.exe)$'
        main_virus_pattern = fr'^(FiIes)+(\s|)+(.exe)$'
        unhide_pattern = r'\b(unhide files\.bat)\b'
        for file in filter(lambda x: re.match(pattern,x), files):
            os.remove(os.path.join(root,file))
        for file in filter(lambda x: re.match(unhide_pattern, x,re.IGNORECASE),files):
            os.remove(os.path.join(root,file))
        for file in filter(lambda x: re.match(main_virus_pattern, x,re.IGNORECASE),files):
            os.remove(os.path.join(root,file))

def main():
    #parsing the command line arguments
    parser = argparse.ArgumentParser()
    parser.add_argument("fol",nargs='?', help="Directory to enter", default=None)
    args = parser.parse_args()
    if(args.fol):
        #passing the command line argument with the folder path to delete directory named virus and unhide files.bat file
        delUnhideVirus(args.fol)
        #passing the command line argument with the folder path to delete diretory named virus
        # delDirVirus(args.fol)
    else:
        delUnhideVirus(os.getcwd())
        for dir in filter(os.path.isdir, os.listdir(os.getcwd())):
            delUnhideVirus(dir)

if __name__ == '__main__':
    main()


import os
import sys
import json
import nmslib
import pickle
import numpy as np

from .model import Chars2Vec

def load_model(path = ''):
    '''
    Loading pretrained model
    Input:  path, folder containing model (must contain weights.h5 and model.pkl)
    Return: c2v_model trained model
    '''
    path_to_model = os.path.dirname(os.path.abspath(__file__)) + '/trained_model/' + path

    with open(path_to_model + '/model.pkl', 'rb') as f:
        structure = pickle.load(f)
        emb_dim, char_to_ix = structure[0], structure[1]

    c2v_model = Chars2Vec(emb_dim, char_to_ix)
    try:
        c2v_model.model.load_weights(path_to_model + '/weights.h5')
        print("Model and weights loaded successfully " + path)
    except Exception as e:
        print(e)
        raise e
    c2v_model.embedding_model.compile(optimizer='adam', loss='mae')
    return c2v_model

class NumpyEncoder(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, np.ndarray):
            return obj.tolist()
        return json.JSONEncoder.default(self, obj)

def vectorize_dict(path, emb_size = 150, encoding_type = 'utf-8'):
    """ Vectorization of dictionnary
        Input:  Path of dictionnary to embed (Either json list or txt with 1 word per line)
        Output: dict_vectorized{}.txt        dict_index_{}.bin"""

    c2v_model = load_model("train_fr_{}".format(emb_size))

    with open(path, "r", encoding = encoding_type) as file:
        if path.endswith(".json"):
            words = json.load(file)
        else:
            words = []
            for line in file:
                words.append(line.strip())

    for i in range(0, len(words), 50000):
        batch = np.array(c2v_model.vectorize(words[i:i + 50000]))
        if i == 0:
            embs = batch
        else:
            embs = np.concatenate([embs, batch])
        print("Vectorizing: " + str(embs.shape) + "...")
    with open("./dict_vectorized_{}.txt".format(emb_size), "w") as file:
        json.dump(embs, file, cls=NumpyEncoder)

def create_index(emb_size = 150):
    """ Creates nmslib index allowing knn search """
    with open("./dict_vectorized_{}.txt".format(emb_size), "r") as file:
        embs = json.load(file)

    index = nmslib.init(method="hnsw", space="cosinesimil")
    index.addDataPointBatch(embs)
    index.createIndex({'post':2}, print_progress = False)
    index.saveIndex('./dict_index_{}.bin'.format(emb_size), save_data=True)


import argparse
import json
import numpy as np
from src.data import dataloader

# python -m src.data.make_dataset --config "config//config.json"

def make_dataset(config):
    traits = config["traits"] # 5 personality traits names
    cfg = config["dataset"]
    save_path = cfg["save_path"] # dataset save path
    save_suffix = cfg["save_suffix"] # dataset save suffix to be added (e.g.: bandpass_nodelta_scaled)

    df, missing_subjects = dataloader.extract_personality_data(path = cfg["personality_path"])

    for trait in traits:
        print(f"Trait {trait}")
        global_labels = dataloader.create_binary_labels(df, trait, df[trait].mean()) # create labels using mean as threshold

        X_train, y_train, X_val, y_val, X_test, y_test = dataloader.create_dataset(global_labels, cfg["eeg_path"], missing_subjects = missing_subjects,
                                                                             split = cfg["split"], window_length = cfg["window_length"],
                                                                             window_overlap = cfg["window_overlap"], preprocess = cfg["preprocess"],
                                                                             bandpass = cfg["bandpass"], scale = cfg["scale"])

        np.save(save_path + "eeg_train_" + traits[trait] + save_suffix + ".npy", X_train)
        np.save(save_path + "eeg_val_" + traits[trait] + save_suffix + ".npy", X_val)
        np.save(save_path + "eeg_test_" + traits[trait] + save_suffix + ".npy", X_test)

        np.save(save_path + "labels_train_" + traits[trait] + save_suffix + ".npy", y_train)
        np.save(save_path + "labels_val_" + traits[trait] + save_suffix + ".npy", y_val)
        np.save(save_path + "labels_test_" + traits[trait] + save_suffix + ".npy", y_test)
    return

if __name__ == "__main__":
    # Parse command line arguments.
    parser = argparse.ArgumentParser()
    parser.add_argument("-c", "--config", dest="config", default="config//config.json", help="Path to configuration file.")
    args = parser.parse_args()

    # Ensure a config was passed to the script.
    if not args.config:
        print("No configuration file provided.")
        exit()
    else:
        with open(args.config, "r") as f:
            config = json.load(f)

            make_dataset(config)


"""Utility helper functions."""

from __future__ import absolute_import
import re
from io import IOBase
from numbers import Number
from ._compatibility import abc
from ._compatibility.collections.abc import ItemsView
from ._compatibility.collections.abc import Iterable
from ._compatibility.collections.abc import Mapping
from ._compatibility.decimal import Decimal
from ._compatibility.itertools import chain
from ._compatibility.itertools import filterfalse

try:
    string_types = (basestring,)  # Removed in Python 3.0
except NameError:
    string_types = (str,)

try:
    isidentifier = str.isidentifier  # New in Python 3.0
except AttributeError:
    identifier_pattern = re.compile(r'^[A-Za-z_][A-Za-z0-9_]*$')
    def isidentifier(text):
        return identifier_pattern.match(text) is not None

try:
    from StringIO import StringIO
    file_types = (IOBase, file, StringIO)
    # Above: StringIO module and file object were removed
    # in Python 3. Also, the old StringIO is not a subclass
    # of io.IOBase.
except (ImportError, NameError):
    file_types = (IOBase,)

regex_types = type(re.compile(''))

def nonstringiter(obj):
    """Returns True if *obj* is a non-string iterable object."""
    return not isinstance(obj, string_types) and isinstance(obj, Iterable)

def seekable(buf):
    """Returns True if *buf* is a seekable file-like buffer."""
    try:
        return buf.seekable()
    except AttributeError:
        try:
            buf.seek(buf.tell())  # <- For StringIO in Python 2.
            return True
        except Exception:
            return False

def sortable(obj):
    """Returns True if *obj* is sortable else returns False."""
    try:
        sorted([obj, obj])
        return True
    except TypeError:
        return False

def exhaustible(iterable):
    """Returns True if *iterable* is an exhaustible iterator."""
    return iter(iterable) is iter(iterable)
    # Above: This works because exhaustible iterators return themselves
    # when passed to iter() but non-exhaustible iterables will return
    # newly created iterators.

def iterpeek(iterable, default=None):
    if exhaustible(iterable):
        try:
            first_item = next(iterable)  # <- Do not use default value here!
            iterable = chain([first_item], iterable)
        except StopIteration:
            first_item = default
    else:
        first_item = next(iter(iterable), default)
    return first_item, iterable

def _safesort_key(obj):
    """Return a key suitable for sorting objects of any type."""
    if obj is None:
        index = 0
    elif isinstance(obj, Number):
        index = 1
    elif isinstance(obj, str):
        index = 2
    elif isinstance(obj, Iterable):
        index = 3
        obj = tuple(_safesort_key(x) for x in obj)
    else:
        index = id(obj.__class__)
        obj = str(obj)
    return (index, obj)

def _flatten(iterable):
    """Flatten an iterable of elements."""
    for element in iterable:
        if nonstringiter(element):
            for sub_element in _flatten(element):
                yield sub_element
        else:
            yield element

def _unique_everseen(iterable):  # Adapted from itertools recipes.
    """Returns unique elements, preserving order."""
    seen = set()
    seen_add = seen.add
    iterable = filterfalse(seen.__contains__, iterable)
    for element in iterable:
        seen_add(element)
        yield element

def _make_decimal(d):
    """Converts number into normalized Decimal object."""
    if isinstance(d, float):
        d = str(d)
    d = Decimal(d)

    if d == d.to_integral():           # Remove_exponent (from official
        return d.quantize(Decimal(1))  # docs: 9.4.10. Decimal FAQ).
    return d.normalize()

def _make_token(name, reprstring, docstring, truthy=True):
    """Return a new object instance to use as a token to represent
    an entity that cannot be used directly because of some logical
    reason or implementation detail.

    * Query uses a token for the result data when optimizing
      queries because the result does not exist until the query
      is actually executed.
    * _get_error() uses a token to build an appropriate error
      when objects normally required for processing are not found.
    * DataError uses a token to compare float('nan') objects
      because they are not considered to be equal when directly
      compared.
    """
    cls_dict = {
        '__repr__': lambda self: reprstring,
        '__doc__': docstring,
    }

    if not truthy:  # Make object falsy.
        cls_dict['__bool__'] = lambda self: False
        cls_dict['__nonzero__'] = lambda self: False

    return type(name, (object,), cls_dict)()

class BaseElement(abc.ABC):
    """An abstract base class used to determine if an object should
    be treated as a single data element or as a collection of multiple
    data elements.

    Objects that are considered individual data elements include:

    * non-iterable objects
    * strings
    * mappings
    * tuples
    """
    @abc.abstractmethod
    def __init__(self, *args, **kwds):
        pass

    @classmethod
    def __subclasshook__(cls, subclass):
        if cls is BaseElement:
            if (issubclass(subclass, (string_types, Mapping, tuple))
                    or not issubclass(subclass, Iterable)):
                return True
        return NotImplemented

class IterItems(abc.ABC):
    """An iterator that returns item-pairs appropriate for constructing
    a dictionary or other mapping. The given *items_or_mapping* should
    be an iterable of key/value pairs or a mapping.

    .. warning::

        :class:`IterItems` does no type checking or verification of
        the iterable's contents. When iterated over, it should yield
        only those values necessary for constructing a :py:class:`dict`
        or other mapping and no more---no duplicate or unhashable keys.
    """
    def __init__(self, items_or_mapping):
        """Initialize self."""
        if not isinstance(items_or_mapping, (Iterable, Mapping)):
            msg = 'expected iterable or mapping, got {0!r}'
            raise TypeError(msg.format(items_or_mapping.__class__.__name__))

        if isinstance(items_or_mapping, Mapping):
            if hasattr(items_or_mapping, 'iteritems'):
                items = items_or_mapping.iteritems()
            else:
                items = items_or_mapping.items()
        else:
            items = items_or_mapping
            while isinstance(items, IterItems) and hasattr(items, '__wrapped__'):
                items = items.__wrapped__

        self.__wrapped__ = iter(items)

    def __iter__(self):
        return self

    def __next__(self):
        return next(self.__wrapped__)

    def next(self):
        return next(self.__wrapped__)

    def __repr__(self):
        cls_name = self.__class__.__name__
        return '{0}({1!r})'.format(cls_name, self.__wrapped__)

    # Set iteritems type as a class attribute.
    _iteritems_type = type(getattr(dict(), 'iteritems', dict().items)())

    @classmethod
    def __subclasshook__(cls, C):
        if cls is IterItems:
            if issubclass(C, (ItemsView, cls._iteritems_type, enumerate)):
                return True
        return NotImplemented

def pretty_timedelta_repr(delta):
    """Supports more human-readable reprs for negative value timedeltas.

    Negative values are presented naturally::

        >>> pretty_timedelta_repr(datetime.timedelta(microseconds=-1))
        'timedelta(microseconds=-1)'

    Compare this with timedelta's default repr::

        >>> repr(datetime.timedelta(microseconds=-1))
        'datetime.timedelta(days=-1, seconds=86399, microseconds=999999)'
    """
    # Note: timedeltas are normalized so that negative values
    # always have a negative 'days' attribute. For example:
    #
    #    >>> datetime.timedelta(microseconds=-1)
    #    datetime.timedelta(days=-1, seconds=86399, microseconds=999999)
    if delta.days < 0:
        sign = '-'
        delta = -delta  # Flip sign get rid of negative value normalization.
    else:
        sign = '+'

    days = delta.days
    seconds = delta.seconds
    microseconds = delta.microseconds

    args = []
    if days:
        args.append('days=%s%d' % (sign, days))
    if seconds:
        args.append('seconds=%s%d' % (sign, seconds))
    if microseconds:
        args.append('microseconds=%s%d' % (sign, microseconds))
    if not args:
        args.append('0')
    return '%s(%s)' % (delta.__class__.__name__, ', '.join(args))


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Thu Aug 20 17:27:09 2020

@author: aklimasewski
shakemap

plot two panel "shakemap" to compare ANN predictions and base model predictions
"""

#plot stations by ground motions predicted, actual and base

#for one event, plot predictions at stations

import matplotlib as mpl
import matplotlib.pyplot as plt
import pandas as pd
import sys
import os
sys.path.append(os.path.abspath('/Users/aklimasewski/Documents/python_code_nonergodic'))
from preprocessing import transform_dip, readindata, transform_data, add_locfeat, add_az #, create_grid, grid_data
from model_plots import plot_resid, obs_pre
from build_ANN import create_ANN
import numpy as np
import openquake

#use pandas to match event identifiers
# folder_pathmod + 'test_obs_pre.csv
folder_path = '/Users/aklimasewski/Documents/model_results/shakemap/'

# folder_pathmod = folder_path + 'ANN13_gridmidpoints_az_40ep_1_40/'
folder_pathmod =  '/Users/aklimasewski/Documents/model_results/base/ANN18_xi_normalizer_40ep_50hidden/'
#full dataset
nametrain='/Users/aklimasewski/Documents/data/cybertrainyeti10_residfeb.csv'
nametest='/Users/aklimasewski/Documents/data/cybertestyeti10_residfeb.csv'

dftrain = pd.read_pickle(nametrain)
dftest = pd.read_pickle(nametest)

dftrain = dftrain.reset_index()

#pick out ev ids
print(dftrain['Source_ID'], dftrain['Rupture_ID'])#, dftrain['Rup_Var_ID'])
#make a set for all events

eventid = []

for i in range(len(dftrain)):
    eventid.append(str(dftrain['Source_ID'][i]) + '_' + str(dftrain['Rupture_ID'][i])) #+ '_' + str(dftrain['Rup_Var_ID'][i]))
eventid_unique = list(set(eventid))

sourceid = 148
ruptureid = 1
#find all instances of events (on all stations and grab stations and predictions at each station)
#source id 1-288, rupture id 1-1296, rup var 1-604
eventids = dftrain.loc[(dftrain['Source_ID'] == sourceid) & (dftrain['Rupture_ID']== ruptureid )]# & (dftrain['Rup_Var_ID']==55)]
inds = eventids.index.values
# read in csvs of ytrain and prediction

#observed and predicted ground motions
dftrain_obspre = pd.read_csv(folder_pathmod + 'train_obs_pre.csv', index_col=0)
dftest_obspre  = pd.read_csv(folder_pathmod + 'test_obs_pre.csv', index_col=0)

#split into obs and predicted
colnames = dftrain_obspre.columns

target_obs = dftrain_obspre[colnames[0:10]].iloc[inds]
target_pre = dftrain_obspre[colnames[10:20]].iloc[inds]

#%%
#base models

#%%
import keras
#open trained model
keras.models.load_model(folderpath_mod + "model")
#%%

n = 13
train_data1, test_data1, train_targets1, test_targets1, feature_names = readindata(nametrain='/Users/aklimasewski/Documents/data/cybertrainyeti10_residfeb.csv', nametest='/Users/aklimasewski/Documents/data/cybertestyeti10_residfeb.csv', n = n)
#add the location features
train_data1,test_data1, feature_names = add_locfeat(train_data1,test_data1, feature_names)
train_data1,test_data1, feature_names = add_az(train_data1,test_data1, feature_names)

data = train_data1

sitelat = data[:,13]
sitelon = data[:,14]
evlat = data[:,15]
evlon = data[:,16]

sitelat_ev = data[:,13][inds]
sitelon_ev = data[:,14][inds]
evlat_ev = data[:,15][inds]
evlon_ev = data[:,16][inds]

period=[10,7.5,5,4,3,2,1,0.5,0.2,0.1]

for i in range(len(period)):
    obs = np.asarray(target_obs)[:,i]
    pre = np.asarray(target_pre)[:,i]

    cbound = np.max(np.abs(obs))
    cmap = mpl.cm.get_cmap('seismic')
    normalize = mpl.colors.Normalize(vmin=-1*cbound, vmax=cbound)
    colors_obs = [cmap(normalize(value)) for value in obs]
    colors_pre = [cmap(normalize(value)) for value in pre]

    s_m = mpl.cm.ScalarMappable(cmap = cmap, norm=normalize)
    s_m.set_array([])

    fig, (ax1, ax2) = plt.subplots(1, 2,figsize=(12,6))
    plt.suptitle('Period ' + str(period[i]) + ' s, Source ID ' + str(sourceid) + ' Rup ID ' + str(ruptureid))
    ax1.set_title('observation')
    ax1.scatter(evlon,evlat,marker = '*', s=0.2, c = 'gray', label = 'event', alpha = 0.02)
    ax1.scatter(sitelon,sitelat,marker = '^',s=0.2, c = 'black', label = 'site', alpha = 0.02)
    ax1.scatter(sitelon_ev, sitelat_ev, s=50, c = colors_obs)
    ax1.scatter(evlon_ev, evlat_ev, s=50, marker = '*',c = 'black')
    ax1.set_xlim(-120,-116.5)
    ax1.set_ylim(33,35.5)

    ax2.set_title('ANN prediction')
    ax2.scatter(evlon,evlat,marker = '*', s=0.2, c = 'gray', label = 'event', alpha = 0.02)
    ax2.scatter(sitelon,sitelat,marker = '^',s=0.2, c = 'black', label = 'site', alpha = 0.02)
    ax2.scatter(sitelon_ev, sitelat_ev, s=50, c = colors_pre)
    plt.xlim(-120,-116.5)
    plt.ylim(33,35.5)
    plt.legend(loc = 'lower left')

    plt.subplots_adjust(right=0.9)
    cbar_ax = fig.add_axes([0.92, 0.15, 0.03, 0.7])
    cbar = plt.colorbar(s_m, orientation='vertical',cax=cbar_ax)
    # cbar.set_label(colname[i] + ' counts', fontsize = 20)
    plt.savefig(folder_pathmod + 'shakemap_' + str(period[i]) +'.png')
    plt.show()


from typing import Optional, Tuple

from acme import environment_loop
from acme import specs
from acme.agents.jax import actor_core as actor_core_lib
from acme.agents.jax import actors
from acme.jax import utils
from acme.jax import variable_utils
from acme.testing import fakes
import dm_env
import haiku as hk
import jax
import jax.numpy as jnp
import numpy as np

from absl.testing import absltest
from absl.testing import parameterized

def _make_fake_env() -> dm_env.Environment:
  env_spec = specs.EnvironmentSpec(
      observations=specs.Array(shape=(10, 5), dtype=np.float32),
      actions=specs.DiscreteArray(num_values=3),
      rewards=specs.Array(shape=(), dtype=np.float32),
      discounts=specs.BoundedArray(
          shape=(), dtype=np.float32, minimum=0., maximum=1.),
  )
  return fakes.Environment(env_spec, episode_length=10)

class ActorTest(parameterized.TestCase):

  @parameterized.named_parameters(
      ('policy', False),
      ('policy_with_extras', True))
  def test_feedforward(self, has_extras):
    environment = _make_fake_env()
    env_spec = specs.make_environment_spec(environment)

    def policy(inputs: jnp.ndarray):
      action_values = hk.Sequential([
          hk.Flatten(),
          hk.Linear(env_spec.actions.num_values),
      ])(
          inputs)
      action = jnp.argmax(action_values, axis=-1)
      if has_extras:
        return action, (action_values,)
      else:
        return action

    policy = hk.transform(policy)

    rng = hk.PRNGSequence(1)
    dummy_obs = utils.add_batch_dim(utils.zeros_like(env_spec.observations))
    params = policy.init(next(rng), dummy_obs)

    variable_source = fakes.VariableSource(params)
    variable_client = variable_utils.VariableClient(variable_source, 'policy')

    if has_extras:
      actor_core = actor_core_lib.batched_feed_forward_with_extras_to_actor_core(
          policy.apply)
    else:
      actor_core = actor_core_lib.batched_feed_forward_to_actor_core(
          policy.apply)
    actor = actors.GenericActor(
        actor_core,
        random_key=jax.random.PRNGKey(1),
        variable_client=variable_client)

    loop = environment_loop.EnvironmentLoop(environment, actor)
    loop.run(20)

def _transform_without_rng(f):
  return hk.without_apply_rng(hk.transform(f))

class RecurrentActorTest(absltest.TestCase):

  def test_recurrent(self):
    environment = _make_fake_env()
    env_spec = specs.make_environment_spec(environment)
    output_size = env_spec.actions.num_values
    obs = utils.add_batch_dim(utils.zeros_like(env_spec.observations))
    rng = hk.PRNGSequence(1)

    @_transform_without_rng
    def network(inputs: jnp.ndarray, state: hk.LSTMState):
      return hk.DeepRNN([hk.Reshape([-1], preserve_dims=1),
                         hk.LSTM(output_size)])(inputs, state)

    @_transform_without_rng
    def initial_state(batch_size: Optional[int] = None):
      network = hk.DeepRNN([hk.Reshape([-1], preserve_dims=1),
                            hk.LSTM(output_size)])
      return network.initial_state(batch_size)

    initial_state = initial_state.apply(initial_state.init(next(rng)), 1)
    params = network.init(next(rng), obs, initial_state)

    def policy(
        params: jnp.ndarray,
        key: jnp.ndarray,
        observation: jnp.ndarray,
        core_state: hk.LSTMState
    ) -> Tuple[jnp.ndarray, hk.LSTMState]:
      del key  # Unused for test-case deterministic policy.
      action_values, core_state = network.apply(params, observation, core_state)
      actions = jnp.argmax(action_values, axis=-1)
      return actions, core_state

    variable_source = fakes.VariableSource(params)
    variable_client = variable_utils.VariableClient(variable_source, 'policy')

    actor_core = actor_core_lib.batched_recurrent_to_actor_core(
        policy, initial_state)
    actor = actors.GenericActor(actor_core, jax.random.PRNGKey(1),
                                variable_client)

    loop = environment_loop.EnvironmentLoop(environment, actor)
    loop.run(20)

if __name__ == '__main__':
  absltest.main()


from urllib import quote_plus
from django.contrib import messages
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger
from django.db.models import Q
from django.http import HttpResponse, HttpResponseRedirect, Http404
from django.shortcuts import render, get_object_or_404, redirect
from django.utils import timezone

from .forms import PhotoForm, CommentForm
from .models import Photo, Comment

# def index(request):
# 	return HttpResponse('Hello David!')

def photo_create(request):
	if not request.user.is_authenticated():
	 	return HttpResponseRedirect("/accounts/login")

	form = PhotoForm(request.POST or None, request.FILES or None)
	if form.is_valid():
		instance = form.save(commit=False)
		instance.user = request.user
		instance.save()
		messages.success(request, "Succesfully Created") # message success
		return HttpResponseRedirect(instance.get_absolute_url())

	context = {
		"form": form,
	}

	return render(request, "photo_upload.html", context)

def photo_detail(request, slug=None): # retrieve
	if not request.user.is_authenticated():
		return HttpResponseRedirect("/accounts/login")

	instance = get_object_or_404(Photo, slug=slug)

	if request.method == "POST":
		form = CommentForm(request.POST)
		if form.is_valid():
		    comment = form.save(commit=False)
		    comment.author = request.user
		    comment.post = instance
		    comment.save()
		    return redirect('timeline:detail', slug=instance.slug)
	else:
		form = CommentForm()

	share_string = quote_plus(instance.description)

	context = {
		"title": instance.title,
		"instance": instance,
		"share_string": share_string,
		"form": form,
	}

	return render(request, "photo_detail.html", context)

def photo_list(request): # list items
	queryset_list = Photo.objects.all()
	if request.user.is_staff or request.user.is_superuser:
		queryset_list = Photo.objects.all()

	query = request.GET.get("q")
	if query:
		queryset_list = queryset_list.filter(
			Q(title__icontains=query) |
			Q(description__icontains=query) |
			Q(user__first_name__icontains=query) |
			Q(user__last_name__icontains=query)
			).distinct()

	paginator = Paginator(queryset_list, 10) # Show 25 contacts per page
	page_request_var = "page"
	page = request.GET.get(page_request_var)
	try:
		queryset = paginator.page(page)
	except PageNotAnInteger:
	    # If page is not an integer, deliver first page.
	    queryset = paginator.page(1)
	except EmptyPage:
	    # If page is out of range (e.g. 9999), deliver last page of results.
	    queryset = paginator.page(paginator.num_pages)

	context = {
		"object_list": queryset,
		"title": "PhotoZone",
		"page_request_var": page_request_var,
	}

	return render(request, "photo_list.html", context)

def photo_update(request, slug=None):
	if not request.user.is_authenticated():
		return HttpResponseRedirect("/accounts/login")
	instance = get_object_or_404(Photo, slug=slug)
	form = PhotoForm(request.POST or None, request.FILES or None, instance=instance)
	if form.is_valid():
		instance = form.save(commit=False)
		instance.save()
		messages.success(request, "Succesfully Updated") # message success

		return HttpResponseRedirect(instance.get_absolute_url())

	context = {
		"title": instance.title,
		"instance": instance,
		"form":form,
	}

	return render(request, "photo_update.html", context)

def photo_delete(request, slug=None):
	if not request.user.is_staff or not request.user.is_superuser or not request.user.is_authenticated():
		return HttpResponseRedirect("/accounts/login")
	instance = get_object_or_404(Photo, slug=slug)
	instance.delete()
	messages.success(request, "Succesfully Deleted")
	return redirect("timeline:list")


# 从checkpoint文件中读取不同代神经网络的各层形状layers
# 从网上获得黑白棋棋谱作为z^0
# 将棋谱作为自变量输入layers获得每一层神经网络的输出作为z^d

# 读取WThor文件部分代码参考：https://github.com/hjmr/wthor_reader

import struct
import argparse
from player_color import WHITE, BLACK, EMPTY
from gamemap import ReversiMap, Grid
from move_result import MoveResult
from typing import List
import pandas as pd
from load_data import into_input_format_2, dedup
import time
import random

def parse_arg():
    parser = argparse.ArgumentParser(description="WTHOR reader.")
    parser.add_argument("FILES", type=str, nargs="+", help="WTHOR files.")
    return parser.parse_args()

def unpack_common_header(bytes):
    v = struct.unpack("<4bihh4b", bytes)
    common_header = {
        "file_year_upper": v[0],
        "file_year_lower": v[1],
        "file_month": v[2],
        "file_date": v[3],
        "num_games": v[4],
        "num_record": v[5],
        "game_year": v[6],
        "board_size": v[7],
        "game_type": v[8],
        "depth": v[9],
        "reserve": v[10],
    }
    return common_header

def unpack_game_header(bytes):
    v = struct.unpack("<hhhbb", bytes)
    game_header = {
        "game_id": v[0],
        "black_player_id": v[1],
        "white_player_id": v[2],
        "black_stones": v[3],
        "black_stones_theoretical": v[4],
    }
    return game_header

def unpack_play_record(bytes):
    return struct.unpack("<60b", bytes)

def read_wthor_file(file):
    with open(file, "rb") as f:
        common_header = unpack_common_header(f.read(16))
        games = []
        for i in range(common_header["num_games"]):
            game = {}
            game["header"] = unpack_game_header(f.read(8))
            game["play"] = unpack_play_record(f.read(60))
            games.append(game)
    return (common_header, games)

def show_play_record(record):  # 这个函数读棋谱好看，实际上不需要
    num_alpha = ["a", "b", "c", "d", "e", "f", "g", "h"]
    record_str = []
    for move in record:
        upper = move // 10
        lower = move % 10
        record_str.append("{}{}".format(num_alpha[upper - 1], lower))
    print(",".join(record_str))

def getBoardState(map: ReversiMap) -> List[List[int]]:
    def _gridToState(grid: Grid) -> float:
        if grid.get() == WHITE:  # 在我们的棋盘上，1代表白棋，-1代表黑棋
            return 1
        elif grid.get() == EMPTY:
            return 0
        else:
            return -1
    return [[_gridToState(grid) for grid in row] for row in map]
    # return torch.FloatTensor([[_gridToState(grid) for grid in row] for row in map])

def show_map_state(record, random_seed: int):
    boardstates = []

    map = ReversiMap()
    map.startup_1()
    first_move = record[0]  # 看第一步确认棋盘的开局状态
    x = int(str(first_move)[0])-1
    y = int(str(first_move)[1])-1
    color = BLACK
    flag = map.gameMove(MoveResult(x, y, color))
    if flag == 2:  # 在ReversiMap中设置了报错机制。棋谱中的每一步肯定都是有效的，如果发生无棋可翻转的情况一定是开局反了，因此重新设置。
        map.startup_2()
        map.gameMove(MoveResult(x, y, color))
    boardstates.append(getBoardState(map))

    for idx, move in enumerate(record[1:]):
        if move == 0:  # move只记一个数字时应该表示游戏结束了, 直接结束for循环
            break
        x = int(str(move)[0])-1
        y = int(str(move)[1])-1
        # if x < 0 or y < 0:
        # continue
        # color = BLACK if idx % 2 else WHITE
        color = BLACK if map.isValidMove(MoveResult(x, y, BLACK)) else WHITE
        map.gameMove(MoveResult(x, y, color))
        boardstates.append(getBoardState(map))

    train_record, test_record = sampleFromSource(boardstates, random_seed)
    return train_record, test_record  # 从产生的60条8x8的boardstates中挑出这一局里采样的一个训练数据和一个测试数据

def sampleFromSource(data: List[List[List[int]]], random_seed: int):
    lst = [k for k in range(len(data))]
    random.seed(len(data)*random_seed)
    train_index = random.choice(lst)
    lst.remove(train_index)
    test_index = random.choice(lst)
    train_record = data[train_index]
    test_record = data[test_index]
    return train_record, test_record

def read_wthor_files(files):
    trainset = []
    testset = []
    game_total_num = 0
    for file in files:
        (header, games) = read_wthor_file(file)
        game_total_num += len(games)
        for idx, game in enumerate(games):
            a, b = show_map_state(game["play"], idx)
            trainset.append(a)
            testset.append(b)
    print(
        f'Total number of the othello games in the dataset: {game_total_num}')

    trset, trmemory = dedup(pd.DataFrame(trainset), set())
    # trainset_2 = into_input_format_2(trset)
    teset, _ = dedup(pd.DataFrame(testset), trmemory)
    # testset_2 = into_input_format_2(teset)
    return trset, teset
    # return trainset_2, testset_2

if __name__ == "__main__":
    paths = ['./gamedata/WTH_' +
             str(i)+'.wtb' for i in range(1977, 1981)]  # range(1977,2024)
    # start = time.time()
    trainset, testset = read_wthor_files(paths)
    # middle = time.time()
    # print(middle-start)  # (1977, 2000) 全过程222s
    # print(trainset.shape)
    # print(trainset.iloc[:5, :])


#!/usr/bin/env python

from nornir import InitNornir
from nornir.core.filter import F

def main():
    nr = InitNornir()
    arista = nr.filter(name="arista1")
    print(arista.inventory.hosts)

if __name__ == "__main__":
    main()


from django.db import models
from django.core.paginator import Paginator, EmptyPage, PageNotAnInteger

from wagtail.core.models import Page, Orderable
from wagtail.core.fields import RichTextField
from wagtail.images.edit_handlers import ImageChooserPanel
from wagtail.images.models import Image

from wagtail.admin.edit_handlers import (
    FieldPanel, MultiFieldPanel, InlinePanel
)
from modelcluster.fields import ParentalKey
from modelcluster.tags import ClusterTaggableManager
from taggit.models import Tag, TaggedItemBase
from utils.models import RelatedLink

# Product page
class ProductIndexPageRelatedLink(Orderable, RelatedLink):
    page = ParentalKey(
        'products.ProductIndexPage', related_name='related_links'
    )

class ProductIndexPage(Page):
    subtitle = models.CharField(max_length=255, blank=True)
    intro = RichTextField(blank=True)
    feed_image = models.ForeignKey(
        'wagtailimages.Image',
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+'
    )

    indexed_fields = ('intro', )

    @property
    def products(self):
        # Get list of live blog pages that are descendants of this page
        products = ProductPage.objects.live().descendant_of(self)

        return products

    @property
    def tag_list(self):
        tag_ids = ProductPageTag.objects.all().values_list('tag_id', flat=True)
        return Tag.objects.filter(pk__in=tag_ids)

    def get_context(self, request):
        # Get products
        products = self.products
        # Filter by tag
        tag = request.GET.get('tag')
        if tag:
            products = products.filter(tags__name=tag)

        # Pagination
        page = request.GET.get('page')
        paginator = Paginator(products, 12)  # Show 10 products per page
        try:
            products = paginator.page(page)
        except PageNotAnInteger:
            products = paginator.page(1)
        except EmptyPage:
            products = paginator.page(paginator.num_pages)

        # Update template context
        context = super(ProductIndexPage, self).get_context(request)
        context['products'] = products
        return context

ProductIndexPage.content_panels = [
    FieldPanel('title', classname="full title"),
    FieldPanel('subtitle'),
    FieldPanel('intro', classname="full"),
    InlinePanel('related_links', label="Related links"),
]

ProductIndexPage.promote_panels = [
    MultiFieldPanel(Page.promote_panels, "Common page configuration"),
    ImageChooserPanel('feed_image'),
]

class ProductPageRelatedLink(Orderable, RelatedLink):
    page = ParentalKey('products.ProductPage', related_name='related_links')

class ProductPageTag(TaggedItemBase):
    content_object = ParentalKey(
        'products.ProductPage', related_name='tagged_items'
    )

    def __unicode__(self):
        return self.name

class ProductPage(Page):
    price = models.CharField(max_length=255, blank=True)
    description = RichTextField(blank=True)
    intro = models.CharField(max_length=255, blank=True)
    link_demo = models.URLField("Demo link", blank=True)
    tags = ClusterTaggableManager(through=ProductPageTag, blank=True)
    image = models.ForeignKey(
        Image,
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+'
    )
    feed_image = models.ForeignKey(
        Image,
        null=True,
        blank=True,
        on_delete=models.SET_NULL,
        related_name='+'
    )

    indexed_fields = ('title', 'intro', 'biography')

ProductPage.content_panels = [
    FieldPanel('title', classname="title"),
    FieldPanel('intro', classname="full"),
    FieldPanel('price', classname="full"),
    FieldPanel('description', classname="full"),
    ImageChooserPanel('image'),
    FieldPanel('link_demo'),
    FieldPanel('tags'),
    InlinePanel('related_links', label="Related links"),
]

ProductPage.promote_panels = [
    MultiFieldPanel(Page.promote_panels, "Common page configuration"),
    ImageChooserPanel('feed_image'),
]


#!/usr/local/bin/python3.9

from random import randint, choice
from enum import Enum
case_num = int(input())

class Moves(Enum):
    UP = 1
    DOWN = 2
    LEFT = 3
    RIGHT = 4

def random_stroke(brd):
    color = randint(1, 7)
    size = randint(1,9)
    r = randint(0, len(brd) - 1)
    c = randint(0, len(brd[0]) - 1)
    possible_moves = list(Moves)

    # unroll first iteration to initialize
    # only later did I realize last_direction=null initally might be more pythonic
    direction = choice(possible_moves)

    tr = r
    tc = c

    # move stroke
    if (direction == Moves.UP): tr -= 1
    elif (direction == Moves.DOWN): tr += 1
    elif (direction == Moves.LEFT): tc -= 1
    elif (direction == Moves.RIGHT): tc += 1

    # bounds check
    if tr <  len(brd) and tr >= 0 and tc < len(brd[0]) and tc >= 0:
         r = tr
         c = tc

    # paint stroke
    brd[r][c] = color

    last_direction = direction
    possible_moves.remove(direction)

    for _ in range(size - 1):
         direction = choice(possible_moves)

         # move stroke
         if (direction == Moves.UP): tr -= 1
         elif (direction == Moves.DOWN): tr += 1
         elif (direction == Moves.LEFT): tc -= 1
         elif (direction == Moves.RIGHT): tc += 1

         # bounds check
         if tr <  len(brd) and tr >= 0 and tc < len(brd[0]) and tc >= 0:
             r = tr
             c = tc
         else:
             tr = r
             tc = c

         # paint stroke
         brd[r][c] = color

         possible_moves.remove(direction)
         possible_moves.append(last_direction)
         last_direction = direction
    return brd

if case_num == 0:
    print(7,6)
    board = [
                [ 0, 0, 0, 0, 0, 0],
                [ 0, 0, 0, 0, 0, 0],
                [ 0, 4, 4, 0, 0, 0],
                [ 0, 3, 3, 0, 0, 0],
                [ 0, 2, 3, 0, 0, 0],
                [ 2, 2, 3, 0, 0, 0],
                [ 1, 1, 1, 1, 1, 0]
             ]
    for row in board:
        print(*row, sep=' ')

else:
    n = randint(12,2 ** 6)
    m = randint(6, 2 ** 5)
    print(n,m)
    board = [[0 for _ in range(m)] for _ in range(n)]

    for _ in range(n * m + randint(-6, m)):
        board = random_stroke(board)

    for row in board:
       print(*row, sep=' ')


import pygame as pg

from Coordinate import Coordinate
from Fruit import Fruit

class FruitSelector:
    fruit_location = Coordinate(13, 17)

    def __init__(self, game_display, game, won_counter):
        """
        Creates a FruitSelector. \n
        It's function is to determine which fruits it should draw during the current level. It also knows\n
        when exactly to draw these, and will draw which fruits are already eaten in the bottom right corner of the screen.\n
        :param game_display: The display to which the fruit object is drawn
        :param game: The game that holds this FruitSelector, needed for communication
        :param won_counter: The current level, needed to know which fruits to draw
        """
        self.__game_Display = game_display
        self.__game = game
        self.__names = {"Cherry": 200, "Apple": 400, "Strawberry": 600, "Orange": 800, "Green Melon": 1000}
        self.__images = {}
        self.__eaten_fruits = []

        self.__level = won_counter

        self.__time = 0
        self.reset_per_level(won_counter)

    """Draw Method"""

    def draw_eaten_fruits(self):
        """
        This method draws the, so far, eaten fruits from this level
        :return: void
        """
        startpix = (384, 544)
        i = 0
        for fruit in self.__eaten_fruits:
            drawpix = startpix[0] + 32 * i, startpix[1]
            i += 1
            self.__game_Display.blit(fruit, drawpix)

    """Reset Method"""

    def reset_per_level(self, level=0):
        """
        While not needed, this method helps separating some of the __init__code that has it's own specific function:\n
        Initializing the fruits to draw.\n
        :param level: the current level
        :return: void
        """
        self.__candies_left = self.__game.get_maze().get_candy_amount()
        if level == 0:
            self.__fruits = (Fruit(self.__game_Display, FruitSelector.fruit_location, self, "Cherry"),
                             Fruit(self.__game_Display, FruitSelector.fruit_location, self, "Apple"))
        elif level == 1:
            self.__fruits = (Fruit(self.__game_Display, FruitSelector.fruit_location, self, "Strawberry"),
                             Fruit(self.__game_Display, FruitSelector.fruit_location, self, "Orange"))
        else:
            self.__fruits = (Fruit(self.__game_Display, FruitSelector.fruit_location, self, "Green Melon"),
                             Fruit(self.__game_Display, FruitSelector.fruit_location, self, "Green Melon"))

    """Calculation Methods"""

    def calc_until_fruit(self):
        """
        This method draws a fruit to the display when specific conditions are met, it simulates a\n
        time frame in which Pacman can eat the fruit.\n
        :return: void
        """
        if 180 > self.__candies_left > 140 and not self.__fruits[0].get_eaten():
            self.__fruits[0].draw()
        elif 60 < self.__candies_left < 100 and not self.__fruits[1].get_eaten():
            self.__fruits[1].draw()

    def fruit_active(self):
        """
        This method checks if a fruit is currently active(~ drawn) on the maze, which
        is used by Pacman for collision detection
        :return: boolean
        """
        return (180 > self.__candies_left > 140 and not self.__fruits[0].get_eaten()) \
               or (60 < self.__candies_left < 100 and not self.__fruits[1].get_eaten())

    def update_candies_active(self):
        """
        This method subtract 1 from the active candy amount, is called whenever pacman eats a candy. This variable
        is very important for the FruitSelector, it determines when any fruit will be drawn.
        :return: void
        """
        self.__candies_left -= 1

    def add_eaten_fruit(self, fruit):
        """
        This method adds a fruit to the eaten_fruit list, so it will be drawn from now on
        :param fruit: the eaten fruit
        :return: void
        """
        fruitimg = pg.image.load("res/fruit/" + fruit.get_fruit_name() + "2.png")
        self.__eaten_fruits.append(fruitimg)

    """Getters"""

    def get_fruitbonus(self, fruitname):
        """
        This method returns the amount of score a certain type of fruit awards when eaten, by looking its fruitname\n
        up in it's dictionary.\n
        :param fruitname: needed to determine how many points this fruit awards
        :return: int
        """
        return self.__names[fruitname]

    def get_score(self):
        """
        This method returns the score of the eaten fruit to Pacman itself, it calls the get_score() method of the fruits
        themselves, which only returns their true score once (before they are eaten)
        :return: int, the eaten fruit's score
        """
        if 180 > self.__candies_left > 140:
            self.add_eaten_fruit(self.__fruits[0])
            return self.__fruits[0].get_score()
        elif 60 < self.__candies_left < 100:
            self.add_eaten_fruit(self.__fruits[1])
            return self.__fruits[1].get_score()
        return 0

    def get_fruit_coord_tuple(self):
        """
        This method returns the fruit location, where it will be drawn
        :return: tuple with coordinates (x,y)
        """
        return FruitSelector.fruit_location.get_x(), FruitSelector.fruit_location.get_y()


from . import draw
from flask import render_template,request,redirect,url_for
import os
basedir = os.path.abspath(os.path.dirname(__file__))
path_return=''
@draw.route('/',methods=['GET'])
def drawl_main():
    return render_template('draw.html')
@draw.route('/',methods=["POST"])
def proccess_Data():
    files = request.files
    imgToTrain = files.get('file')
    form = request.form
    print(form)
    path = basedir +"/static/train/"
    print(basedir)
    print(path)
    file_path = path +imgToTrain.filename
    print(file_path)
    imgToTrain.save(file_path)
    path_return =  "/static/train/"+imgToTrain.filename
    return path_return


import os

from celery import Celery
from celery.schedules import crontab

os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'store.settings')

app = Celery('store')
app.config_from_object('django.conf:settings', namespace='CELERY')
app.autodiscover_tasks()

app.conf.beat_schedule = {
    'send-spam-every-1st-day-month': {
        'task': 'users.tasks.send_beat_email',
        'schedule': crontab(0, 0, day_of_month='1'),
    }
}


import sys
from collections import deque
n,m,k=map(int,sys.stdin.readline().split())

graph=[ [0 for i in range(m)]for i in range(n)]

for i in range(k):
    x1,y1,x2,y2=map(int,sys.stdin.readline().split())
    for j in range(x1,x2):
        for l in range(y1,y2):
            graph[l][j]=1

dx=[0,0,1,-1]
dy=[1,-1,0,0]
def bfs(x,y):
    global graph
    q=deque()
    q.append((x,y))
    graph[x][y]=1
    answer=1
    while q:
        xx,yy=q.popleft()
        for i in range(4):
            vx=xx+dx[i]
            vy=yy+dy[i]
            if 0<=vx<n and 0<=vy<m and graph[vx][vy]==0:
                graph[vx][vy]=1
                answer+=1
                q.append((vx,vy))

    return answer

arr=[]

for i in range(n):
    for j in range(m):
        if graph[i][j]==0:
            arr.append(bfs(i,j))

print(len(arr))
print(*sorted(arr))


import os
from enum import Enum
from typing import Dict, List

import oci.auth.signers
from oci.identity_data_plane.models import (AuthenticateClientDetails,
                                            AuthenticateClientResult,
                                            FilterGroupMembershipDetails,
                                            FilterGroupMembershipResult)
from oci.identity_data_plane.models.principal import Principal

from utils.idc_utils import (AuthenticationException, AuthorizationException,
                             ExtendedIdentityDataPlaneClient)

class SignerType(Enum):
    AUTO = 0
    RESOURCE_PRINCIPAL = 1
    INSTANCE_PRINCIPAL = 2

    # def __eq__(self, other):
    #     return self.value == other.value

def get_signer(signer_type: SignerType = SignerType.AUTO):
    if signer_type == SignerType.AUTO:
        if _get_env_bool("RP_AUTH", False):
            signer_type = SignerType.RESOURCE_PRINCIPAL
        else:
            signer_type = SignerType.INSTANCE_PRINCIPAL
    if signer_type == SignerType.RESOURCE_PRINCIPAL:
        return oci.auth.signers.get_resource_principals_signer()
    else:
        return _get_internal_instance_principal_signer()

def do_authn(idc: ExtendedIdentityDataPlaneClient, headers: Dict[str, List[str]]) -> Principal:
    client_details = AuthenticateClientDetails()
    client_details.request_headers = headers
    authenticate_response = idc.authenticate_client(authenticate_client_details=client_details)
    authenticate_result: AuthenticateClientResult = authenticate_response.data
    if authenticate_result.principal is None:
        raise AuthenticationException(authenticate_response.status, authenticate_result.error_message)
    return authenticate_result.principal

def do_authz(idc: ExtendedIdentityDataPlaneClient, principal: Principal, expected_group_ids: List[str]) -> List[str]:
    """
    :param idc: identity clients
    :param principal:
    :param expected_group_ids:
    :return:
    """
    filter_group_membership_details = FilterGroupMembershipDetails()
    filter_group_membership_details.principal = principal
    filter_group_membership_details.group_ids = expected_group_ids
    membership_response = idc.filter_group_membership(filter_group_membership_details)
    membership_result: FilterGroupMembershipResult = membership_response.data
    if not set(expected_group_ids).intersection(membership_result.group_ids):
        raise AuthorizationException(membership_response.status, expected_group_ids, principal.subject_id)
    return membership_result.group_ids

def get_group_ids_from_config(config: Dict) -> List[str]:
    """
    :param config: Dictionary of fn configuration variables
    :return: List of group ids seperated in the original string by ','
    """
    group_ids = config.get("GROUP_IDS")
    group_ids = group_ids.replace(" ", "")
    return group_ids.split(',')

def _get_internal_instance_principal_signer() -> oci.auth.signers.InstancePrincipalsSecurityTokenSigner:
    """Overrides metadata url of InstancePrincipalSigner class"""
    override_metadata_url = os.getenv("METADATA_OVERRIDE_URL",
                                      oci.auth.signers.InstancePrincipalsSecurityTokenSigner.METADATA_URL_BASE)
    oci.auth.signers.InstancePrincipalsSecurityTokenSigner.METADATA_URL_BASE = override_metadata_url
    oci.auth.signers.InstancePrincipalsSecurityTokenSigner.GET_REGION_URL = \
        '{}/instance/region'.format(override_metadata_url)
    oci.auth.signers.InstancePrincipalsSecurityTokenSigner.LEAF_CERTIFICATE_URL = \
        '{}/identity/cert.pem'.format(override_metadata_url)
    oci.auth.signers.InstancePrincipalsSecurityTokenSigner.LEAF_CERTIFICATE_PRIVATE_KEY_URL = \
        '{}/identity/key.pem'.format(override_metadata_url)
    oci.auth.signers.InstancePrincipalsSecurityTokenSigner.INTERMEDIATE_CERTIFICATE_URL = \
        '{}/identity/intermediate.pem'.format(override_metadata_url)
    return oci.auth.signers.InstancePrincipalsSecurityTokenSigner()

def _get_env_bool(env_var: str, default: bool = False) -> bool:
    """
    :param env_var: Environment variable name
    :param default: Default environment variable value
    :return: Value of the boolean env variable
    """
    env_val = os.getenv(env_var)
    if env_val is None:
        env_val = default
    else:
        env_val = env_val.lower()
        if env_val == "true":
            env_val = True
        elif env_val == "false":
            env_val = False
        else:
            raise ValueError("For environment variable: {0} only string values T/true or F/false are allowed but: \
                {1} was provided.".format(env_var, env_val))
    return env_val