import os
import numpy as np
import matplotlib.pyplot as plt
from scipy.signal import convolve2d

def load_data(path):
    '''
    Load data from folder data, face images are in the folder facial_images, face features are in the folder facial_features.

    Args:
        path: path of folder data

    Returns:
        imgs: list of face images as numpy arrays
        feats: list of facial features as numpy arrays
    '''
    # List for imgs and feats numpy arrays
    imgs = []
    feats = []

    # List for names of files
    img_files = []
    feat_files = []

    # paths
    img_path = path + '/facial_images'
    feats_path = path + '/facial_features'

    # For img files, make the list of file names in the directory facial_images
    # Using os.walk and os.path methods
    for root, dirs, files in os.walk(img_path, topdown=True):
        for name in files:
            img_files.append(os.path.join(root, name))

    # For img files, let's make python arrays
    for file in img_files:
        with open(file, 'rb') as infile:

            # Let's get width and height from header
            header = infile.readline()
            width, height, maxval = [int(item) for item in header.split()[1:]]

            # Make it into np array with data type uint8 cause the maxval is 255
            imgs.append(np.fromfile(infile, dtype=np.uint8).reshape((height, width)))

    # For feat files, make the list of file names in the directory facial_features
    # Using os.walk and os.path methods
    for root, dirs, files in os.walk(feats_path, topdown=True):
        for name in files:
            feat_files.append(os.path.join(root, name))

    # For feat files, let's make python arrays
    for file in feat_files:
        with open(file, 'rb') as infile:

            # Let's get width and height from header
            # The info of width and height is in second line
            header = infile.readline()
            header = infile.readline()
            width, height = [int(item) for item in header.split()]
            header = infile.readline()

            # Make it into np array with data type uint8 cause the maxval is 255
            feats.append(np.fromfile(infile, dtype=np.uint8).reshape((height, width)))

    return imgs, feats

def gaussian_kernel(fsize, sigma):
    '''
    Define a Gaussian kernel

    Args:
        fsize: kernel size
        sigma: sigma of Gaussian kernel

    Returns:
        The Gaussian kernel
    '''

    # To make the y, x column and rows...
    m = (fsize-1.)/2.

    # Make y, x column and row which has values like -m -m+1 ... m
    # For example, '-2 -1 0 1 2' for size 5
    y,x = np.ogrid[-m:m+1,-m:m+1]

    # Using gaussian
    kernel = np.exp(-(x*x + y*y) / (2.*sigma*sigma))
    # Normalize the values so that the sum of all values be 1
    if kernel.sum() != 0:
        kernel /= kernel.sum()

    return kernel

def downsample_x2(x, factor=2):
    '''
    Downsampling an image by a factor of 2

    Args:
        x: image as numpy array (H * W)

    Returns:
        downsampled image as numpy array (H/2 * W/2)
    '''

    # Getting the height and width from x
    H, W = x.shape
    # Making the empty array sized H/2 * W/2
    downsample = np.empty((H//2, W//2))

    # Just put average of 4 values from 4 places to 1 place
    # Using np.mean method
    # It makes things downsampled by factor of 2
    for i in range(H//2):
        for j in range(W//2):
            # Using numpy mean function
            arr = [x[i*2,j*2],x[i*2+1,j*2],x[i*2,j*2+1],x[i*2+1, j*2+1]]
            downsample[i, j] = np.mean(arr)

    return downsample

def gaussian_pyramid(img, nlevels, fsize, sigma):
    '''
    A Gaussian pyramid is constructed by combining a Gaussian kernel and downsampling.
    Tips: use scipy.signal.convolve2d for filtering image.

    Args:
        img: face image as numpy array (H * W)
        nlevels: number of levels of Gaussian pyramid, in this assignment we will use 3 levels
        fsize: Gaussian kernel size, in this assignment we will define 5
        sigma: sigma of Gaussian kernel, in this assignment we will define 1.4

    Returns:
        GP: list of Gaussian downsampled images, it should be 3 * H * W
    '''
    GP = []

    # Appending first original img
    GP.append(img)

    # For nlevls-1 times
    for i in range(nlevels-1):
        # First, filter with the gaussian kernel
        img = convolve2d(img, gaussian_kernel(fsize, sigma), mode = 'same', boundary = 'symm')
        # And do the downsampling
        img = downsample_x2(img)
        GP.append(img)

    return GP

def template_distance(v1, v2):
    '''
    Calculates the distance between the two vectors to find a match.
    Browse the course slides for distance measurement methods to implement this function.
    Tips:
        - Before doing this, let's take a look at the multiple choice questions that follow.
        - You may need to implement these distance measurement methods to compare which is better.

    Args:
        v1: vector 1
        v2: vector 2

    Returns:
        Distance
    '''
    # SSD, Just subtract two vectors and squares it.
    distance = np.sum((v1-v2)**2)
    # Dot product, using np.sum and np.dot
    #distance = np.sum(np.dot(np.transpose(v1), v2))
    return distance

def sliding_window(img, feat, step=1):
    '''
    A sliding window for matching features to windows with SSDs. When a match is found it returns to its location.

    Args:
        img: face image as numpy array (H * W)
        feat: facial feature as numpy array (H * W)
        step: stride size to move the window, default is 1
    Returns:
        min_score: distance between feat and window
    '''

    min_score = None

    # Getting size of img
    img_h, img_w  = img.shape
    feat_h, feat_w  = feat.shape

    # Go around by pixels and check the distance
    for i in range(img_h):
        # Checking the out of range problem
        if i+feat_h >img_h:
            continue
        for j in range(img_w):
            # Checking the out of range problem
            if j+feat_w >img_w:
                continue
            # Crop the img to feature's size and calculate distance
            val = template_distance(img[i:i+feat_h, j:j+feat_w], feat)
            # Set the value if it's smallest for now
            if min_score == None or val<min_score:
                min_score = val

    return min_score

class Distance(object):

    # choice of the method
    METHODS = {1: 'Dot Product', 2: 'SSD Matching'}

    # choice of reasoning
    REASONING = {
        1: 'it is more computationally efficient',
        2: 'it is less sensitive to changes in brightness.',
        3: 'it is more robust to additive Gaussian noise',
        4: 'it can be implemented with convolution',
        5: 'All of the above are correct.'
    }

    def answer(self):
        '''Provide your answer in the return value.
        This function returns one tuple:
            - the first integer is the choice of the method you will use in your implementation of distance.
            - the following integers provide the reasoning for your choice.
        Note that you have to implement your choice in function template_distance

        For example (made up):
            (1, 1) means
            'I will use Dot Product because it is more computationally efficient.'
        '''
        '''
        Since dot product just multiply every values, it may make things more sensitive to brightness.
        Also, the results seems better when it comes to ssd method.
        '''
        return (2, 2)  # TODO

def find_matching_with_scale(imgs, feats):
    '''
    Find face images and facial features that match the scales

    Args:
        imgs: list of face images as numpy arrays
        feats: list of facial features as numpy arrays
    Returns:
        match: all the found face images and facial features that match the scales: N * (score, g_im, feat)
        score: minimum score between face image and facial feature
        g_im: face image with corresponding scale
        feat: facial feature
    '''
    match = []

    for img in imgs:
        # Initialize the values for the new img
        (score, g_im, feat) = (None, None, None)

        # For each pyramids and feature
        for pyramid in gaussian_pyramid(img, 3, 5, 1.4):

            # print('new_img', pyramid.shape)
            for feat_ in feats:
                # print("feat", feat_.shape)
                # Check the minimum score of this pyramid to feat
                val = sliding_window(pyramid,feat_)
                if val != None:
                # If it is minimum score for all pyramids for now,
                # Set the values to this condition
                    if score == None or val < score:
                        score = val
                        g_im = pyramid
                        feat = feat_

        # The last value is the condition for the minimum score
        if score != None:
            match.append((score, g_im, feat))
            # print(g_im.shape)

    return match


# -*- coding:utf-8 -*-
#创作漫画单篇--》小铅笔--》漫画--》画漫画--》新增单篇--》添加角色--》改变角色方向--》添加素材--》保存
import unittest
from ddt import ddt, data, unpack
from base import base_desired_caps,public_choice,interrupt
from page import createComicPage

@ddt
class MyTestCase(unittest.TestCase):
    def setUp(self):
        base_desired_caps.base_desired_caps(self)
        interrupt.interrupt(self)

    # @data(
    #     ('public', True),
    #     ('draft', True),
    #     ('draft', True),
    #     ('serials',True)
    # )
    @data(
        ('draft', True)
    )
    @unpack
    # 画漫画流程
    def test_creation(self, input,expectedresult):
        exist=createComicPage.createComic(self)
        if input=='draft':
            public_choice.publicAsDraft(self)
        elif input=='public':
            public_choice.publicAsOffice(self)
        elif input == 'serials':
            public_choice.addComicSerials(self)

        try:
            return exist == True
        except Exception:
            print("ERROR")
        self.assertEqual(exist, expectedresult)

    def tearDown(self):
        self.driver.quit()

if __name__ == '__main__':
    unittest.main()


import boto3
from flask import Flask, request, jsonify
from datetime import datetime
import simplejson as json
import os

app = Flask(__name__)
@app.route('/', methods =['GET'])
def predict():
    try:
        dynamodb = boto3.resource(
            'dynamodb',
            region_name='us-east-2'
        )
    except Exception as e:
        raise Exception("Can't get the dynamodb resource: " + str(e))
    table = dynamodb.Table('Weather')
    return str(table.creation_date_time)

if __name__ == '__main__':
    app.run(host='0.0.0.0', port=4000)


import sys, os
parent_dir = os.path.abspath(os.path.dirname(__file__))
vendor_dir = os.path.join(parent_dir, 'vendor')
sys.path.append(vendor_dir)

import logging, datetime, json
import boto3
from functools import partial

# Configure logging
logging.basicConfig()
log = logging.getLogger()
log.setLevel(os.environ.get('LOG_LEVEL','INFO'))
def format_json(data):
  return json.dumps(data, default=lambda d: d.isoformat() if isinstance(d, datetime.datetime) else str(d))

CONTAINER_MAX_MEMORY = int(os.environ.get('CONTAINER_MAX_MEMORY',993))
CONTAINER_MAX_CPU = int(os.environ.get('CONTAINER_MAX_CPU',1024))
TCP_PORT_RESOURCES = [p for p in os.environ.get('TCP_PORT_RESOURCES','').split(',') if p]
UDP_PORT_RESOURCES = [p for p in os.environ.get('UDP_PORT_RESOURCES','').split(',') if p]

# AWS clients
ecs = boto3.client('ecs')
cloudwatch = boto3.client('cloudwatch')

# Returns fully paginated response
def paginated_response(func, result_key, next_token=None):
  args=dict()
  if next_token:
    args['NextToken'] = next_token
  response = func(**args)
  result = response.get(result_key)
  next_token = response.get('NextToken')
  if not next_token:
    return result
  return result + self.paginated_response(func, result_key, next_token)

def describe_ecs_instances(cluster):
  func = partial(ecs.list_container_instances, cluster=cluster)
  ecs_instance_arns = paginated_response(func, 'containerInstanceArns')
  ecs_instances = ecs.describe_container_instances(
      cluster=cluster,
      containerInstances=ecs_instance_arns
    )['containerInstances']
  return [instance for instance in ecs_instances if instance['status'] == 'ACTIVE']

def check_memory(instance):
  return sum(
    resource['integerValue']
    for resource in instance['remainingResources']
    if resource['name'] == 'MEMORY')

def check_cpu(instance):
  return sum(
    resource['integerValue']
    for resource in instance['remainingResources']
    if resource['name'] == 'CPU')

def check_tcp_port(ecs_instances, port):
  return len([
    port
    for instance in ecs_instances
    for resource in instance['remainingResources']
    if resource['name'] == 'PORTS' and port in resource.get('stringSetValue',[])
  ])

def check_udp_port(ecs_instances, port):
  return len([
    port
    for instance in ecs_instances
    for resource in instance['remainingResources']
    if resource['name'] == 'PORTS_UDP' and port in resource.get('stringSetValue',[])
  ])

# Lambda handler function
# Capacity manager needs to calculate two metrics
# 1. The current "spare" container capacity, which is used for scale out (typically when capacity < 1)
# 2. The current "idle host" count, which is used for scale in (typically when idle hosts > 1)
#
def handler(event, context):
  '''
  Sample event for processing
  {
    "account": "543279062384", "region": "us-west-2",
    "detail": {
      "status": "ACTIVE",
      "registeredAt": "2017-07-30T02:36:50.591Z",
      "registeredResources": [
        {"type": "INTEGER", "name": "CPU", "integerValue": 1024},
        {"type": "INTEGER", "name": "MEMORY", "integerValue": 993},
        {"stringSetValue": ["22", "2376", "2375", "51678", "51679"], "type": "STRINGSET", "name": "PORTS"},
        {"stringSetValue": [], "type": "STRINGSET", "name": "PORTS_UDP"}
      ],
      "ec2InstanceId": "i-06cb96155228d49d6",
      "agentConnected": True,
      "containerInstanceArn": "arn:aws:ecs:us-west-2:543279062384:container-instance/0606dc43-fd6a-4837-a4f1-0a5ec2f2d6fe",
      "remainingResources": [
        {"type": "INTEGER", "name": "CPU", "integerValue": 1024},
        {"type": "INTEGER", "name": "MEMORY", "integerValue": 993},
        {"stringSetValue": ["22", "2376", "2375", "51678", "51679"], "type": "STRINGSET", "name": "PORTS"},
        {"stringSetValue": [], "type": "STRINGSET", "name": "PORTS_UDP"}
      ],
      "version": 2,
      "clusterArn": "arn:aws:ecs:us-west-2:543279062384:cluster/microtrader-development-ApplicationCluster-1643NQX67M437",
      "updatedAt": "2017-07-30T02:36:50.629Z",
      "attributes": [],
      "versionInfo": {"agentVersion": "1.14.3", "agentHash": "15de319", "dockerVersion": "DockerVersion: 17.03.1-ce"}
    },
    "detail-type": "ECS Container Instance State Change",
    "source": "aws.ecs",
    "version": "0",
    "time": "2017-07-30T02:36:50Z",
    "id": "d5c8131c-b7c1-44dd-8ab4-d2cf4dc0b736",
    "resources": ["arn:aws:ecs:us-west-2:543279062384:container-instance/0606dc43-fd6a-4837-a4f1-0a5ec2f2d6fe"]
  }
  '''
  log.info("Received event: %s" % format_json(event))

  # STEP 1 - COLLECT RESOURCE DATA
  ecs_cluster = event['detail']['clusterArn']
  # The maximum CPU availble for an idle ECS instance
  ecs_instance_max_cpu = next(
    resource['integerValue']
    for resource in event['detail']['registeredResources']
    if resource['name'] == 'CPU')
  # The maximum memory availble for an idle ECS instance
  ecs_instance_max_memory = next(
    resource['integerValue']
    for resource in event['detail']['registeredResources']
    if resource['name'] == 'MEMORY')

  # Get current container capacity based upon CPU and memory
  ecs_instances = describe_ecs_instances(ecs_cluster)
  cpu_capacity = 0
  memory_capacity = 0
  for instance in ecs_instances:
    cpu_capacity += check_cpu(instance)/CONTAINER_MAX_CPU
    memory_capacity += check_memory(instance)/CONTAINER_MAX_MEMORY
  log.info("Current container cpu capacity of %s" % cpu_capacity)
  log.info("Current container memory capacity of %s" % memory_capacity)

  # Get current container capacity based upon static TCP and UDP ports
  tcp_port_capacity = [len(ecs_instances)]
  udp_port_capacity = [len(ecs_instances)]
  for port in TCP_PORT_RESOURCES:
    tcp_port_capacity.append(len(ecs_instances) - check_tcp_port(ecs_instances, port))
  for port in UDP_PORT_RESOURCES:
    udp_port_capacity.append(len(ecs_instances) - check_udp_port(ecs_instances, port))
  log.info("Current container tcp port capacity of %s" % min(tcp_port_capacity))
  log.info("Current container udp port capacity of %s" % min(udp_port_capacity))
  # STEP 2 - CALCULATE SPARE CONTAINER CAPACITY
  # This is the minimum container capacity count (integer) for CPU, memory and TCP/UDP port resources
  # This metric is used to scale out and typically is triggered with capacity < 1
  container_capacity = min(
    cpu_capacity,
    memory_capacity,
    min(tcp_port_capacity),
    min(udp_port_capacity)
  )
  log.info("Overall container capacity of %s" % container_capacity)

  # STEP 3- CALCULATE IDLE HOST COUNT
  # This is the minimum "virtual" idle host count (float) for CPU, memory and TCP/UDP port resources
  # This metric is used to scale in and typically is triggered with idle host count > 1.0
  idle_hosts = min(
    float(cpu_capacity) / float(ecs_instance_max_cpu / CONTAINER_MAX_CPU),
    float(memory_capacity) / float(ecs_instance_max_memory / CONTAINER_MAX_MEMORY),
    float(min(tcp_port_capacity)),
    float(min(udp_port_capacity))
  )
  log.info("Overall idle host capacity of %s" % idle_hosts)

  # STEP 4 - PUBLISH METRICS
  # Publish container capacity
  cloudwatch.put_metric_data(
    Namespace='AWS/ECS',
    MetricData=[{
      'MetricName': 'ContainerCapacity',
      'Dimensions': [{
        'Name': 'ClusterName',
        'Value': ecs_cluster.split('/')[-1]
      }],
      'Timestamp': datetime.datetime.utcnow(),
      'Value': container_capacity
    }])
  # Publish idle host capacity
  cloudwatch.put_metric_data(
    Namespace='AWS/ECS',
    MetricData=[{
      'MetricName': 'IdleHostCapacity',
      'Dimensions': [{
        'Name': 'ClusterName',
        'Value': ecs_cluster.split('/')[-1]
      }],
      'Timestamp': datetime.datetime.utcnow(),
      'Value': idle_hosts
    }])


import matplotlib.pyplot as plt
import os
import json
import numpy as np
import pandas as pd
import time

import tensorflow as tf
from tensorflow.keras import models
from tensorflow.keras.utils import to_categorical
from sklearn import metrics
from sklearn.preprocessing import Normalizer
from sklearn.metrics import accuracy_score, f1_score
from sklearn.metrics import precision_score, recall_score
from sklearn.metrics import cohen_kappa_score
from sklearn.metrics import hamming_loss
from sklearn.metrics import confusion_matrix
from model_train import TrainModels
from sklearn.preprocessing import label_binarize

from art.attacks.evasion import FastGradientMethod
from art.estimators.classification import KerasClassifier

tf.compat.v1.disable_eager_execution()

class DataSet(object):
    datas: list
    labels: list

class ShowResults(object):
    def __init__(self, dir_name):
        self.epochs = 100
        dir = f'/home/user/ZN/intrusion detection/{dir_name}'
        self.save_dir = os.path.join(dir)
        self.train_path = '/home/user/ZN/intrusion detection/data/train_csv.csv'
        self.test_path = '/home/user/ZN/intrusion detection/data/test_csv.csv'
        self.save_adv_dir = '/home/user/ZN/intrusion detection/history_adv_more'
        self.save_pre_adv_dir = '/home/user/ZN/intrusion detection/history_pre_adv_more'

    def data_pre_process(self, name):
        train_data = pd.read_csv(self.train_path)
        test_data = pd.read_csv(self.test_path)
        test_label = test_data['Label']
        label_list = list(set(tuple(test_label)))
        label_map = {value: str(key) for key, value in enumerate(label_list)}
        for key, value in label_map.items():
            if key == 'Benign':
                label_map[key] = 0
            else:
                label_map[key] = 1
        train_data['Label'] = train_data['Label'].map(label_map)
        test_data['Label'] = test_data['Label'].map(label_map)
        train_data = train_data.dropna(axis=0, how='any')
        test_data = test_data.dropna(axis=0, how='any')
        train_data = np.array(train_data, dtype='float32')
        test_data = np.array(test_data, dtype='float32')
        x_train_inf = np.isinf(train_data)
        train_data[x_train_inf] = 0
        x_test_inf = np.isinf(test_data)
        test_data[x_test_inf] = 0

        x_train = train_data[:, 0:77]
        x_test = test_data[:, 0:77]
        y_train = train_data[:, 78]
        y_test = test_data[:, 78]
        scaler = Normalizer().fit(x_test)
        x_test = scaler.transform(x_test)
        # y_test = to_categorical(y_test)
        if name == 'LSTM':
            x_train = np.reshape(x_train, (x_train.shape[0], 1, x_train.shape[1]))
            x_test = np.reshape(x_test, (x_test.shape[0], 1, x_test.shape[1]))
        elif name == 'GRU':
            x_train = np.reshape(x_train, (x_train.shape[0], 1, x_train.shape[1]))
            x_test = np.reshape(x_test, (x_test.shape[0], 1, x_test.shape[1]))
        elif name == 'CNN':
            x_train = np.reshape(x_train, (x_train.shape[0], x_train.shape[1], 1))
            x_test = np.reshape(x_test, (x_test.shape[0], x_test.shape[1], 1))
        return x_train, x_test, y_train, y_test

    def generate_adv_samples(self, model_path, name):
        name = name
        model = models.load_model(model_path)
        x_train, x_test, y_train, y_test = self.data_pre_process(name)
        classifier = KerasClassifier(clip_values=(np.min(x_train), np.max(x_train)), model=model)
        fgsm = FastGradientMethod(classifier, eps=0.01, minimal=True, eps_step=0.01, num_random_init=35,
                                  targeted=False, batch_size=128)
        x_adv_test = fgsm.generate(x=x_test)
        return x_adv_test

    def get_files_path(self, flag=''):
        path = []
        for _dir in os.listdir(self.save_dir):
            _dir_path = os.path.join(self.save_dir, _dir)
            for file in os.listdir(_dir_path):
                if file.endswith(flag):
                    path.append(os.path.join(self.save_dir, _dir, file))
        return path

    def show_history(self, acc=False):
        for path in self.get_files_path('json'):
            self.history_info(path, acc=acc)

    def history_info(self, path, skip=2, acc=False):
        if acc == True:
            train_flag = 'binary_accuracy'
            val_flag = 'val_binary_accuracy'
            plt.axis([0, self.epochs, 0.7, 1])
        else:
            train_flag = 'loss'
            val_flag = 'val_loss'
            plt.axis([0, self.epochs, 0, 0.5])
        with open(path) as f:
            data = json.loads(f.read())
        train_acc = data[train_flag]
        val_acc = data[val_flag]
        epochs = range(1, len(train_acc) + 1)
        plt.plot(epochs[::skip], train_acc[::skip], 'b', label='train_history')
        plt.plot(epochs[::skip], val_acc[::skip], 'r:', label='val_history')
        # plt.axis([1, 100, 0.7, 1])
        plt.xlabel('epochs')
        plt.ylabel(train_flag)
        plt.grid()
        plt.title(path.split('/')[-2])

        plt.legend()
        plt.show()

    def one_hot(self, index, num=2):
        data = np.zeros(2)
        data[index] = 1
        return data

    def roc(self, model_path, name):
        model = models.load_model(model_path)
        name = name
        print(name)
        x_train, x_test, y_train, y_test = self.data_pre_process(name)
        x_adv_test = self.generate_adv_samples(model_path, name)
        predicts = model.predict(x_test, verbose=1)
        predicts_adv = model.predict(x_adv_test, verbose=1)
        predicts_class = model.predict_classes(x_test, verbose=1)
        predicts_adv_class = model.predict_classes(x_adv_test, verbose=1)
        # for i in range(len(predicts)):
        #     max_value = max(predicts[i])
        #     for j in range(len(predicts[i])):
        #         if max_value == predicts[i][j]:
        #             predicts[i][j] = 1
        #         else:
        #             predicts[i][j] = 0
        # print(x_test.shape, y_test.shape, predicts.shape)
        predicts = np.reshape(predicts, (y_test.shape[0],))
        predicts_adv = np.reshape(predicts_adv, (y_test.shape[0],))
        self.printMetrics(y_test, predicts_class)
        print('------------------------------------')
        self.printMetrics(y_test, predicts_adv_class)
        # print(predicts)
        # auc_results = metrics.roc_auc_score(y_true=y_test, y_score=predicts)
        # print(f'model{model_path}------------')
        # print(f'auc score {auc_results}')

        fpr, tpr, threshold = metrics.roc_curve(y_test, predicts)
        fpr_adv, tpr_adv, threshold_adv = metrics.roc_curve(y_test, predicts_adv)
        print('fpr:', fpr)
        print('fpr_adv', fpr_adv)
        print('threshold', threshold)
        print('threshold_adv', threshold_adv)
        return fpr, tpr, fpr_adv, tpr_adv

    def printMetrics(self, true, pred):
        # print(true[0], pred[0])
        # 分类正确率 true:真实标签 pred:训练标签  normalize: True:分类正确的百分比； False:分类正确的样本数
        # kappa = cohen_kappa_score(true, pred)
        # ham_distance = hamming_loss(true, pred)
        # print('ham_distanc:', ham_distance)
        # print('kappa:', kappa)
        print('Accuracy:', accuracy_score(true, pred))
        # 计算精度precision = TP/(TP+FP) TP:真正例 FP：假正例 average:评价值的平均值的计算方式。可以接收[None, 'binary'(default),'micro','macro','samples','weighted']
        # 'weighted': 相当于类间带权重。各类别的P x 该类别的样本数量（实际值非预测值）/除以样本总数
        print('Precision:', precision_score(true, pred, average='weighted'))
        # 召回率：提取出的正确信息的条数/样本中的信息条数。就是所有准备的条目有多少被检索出来了。
        print('Recall:', recall_score(true, pred, average='weighted'))
        # f1_socre = 2 x (precision x recall)/(precision + recall)
        print('F1-score:', f1_score(true, pred, average='weighted'))
        # 混淆矩阵：
        # 1>列代表预测的类别；行代表实际的类别
        # 2>对角线上的值表示预测正确的数量/比例；非对角线元素是预测错误的部分
        # 混淆矩阵对角线上的值越高越好。在各分类数据的数量不平衡的情况下， 混淆矩阵可以直观的显示分类模型对应各个类别的准确率
        # print('Confusion Matrix:')
        # print(confusion_matrix(true, pred))

    # def metric_infos(self, model_path):
    #     model = models.load_model(model_path)
    #     x_test, y_test = self.data_pre_process()
    #     predicts = model.predict(x_test)
    #     predicts = np.reshape(predicts, (99773,))
    #     tp = 0
    #     tn = 0
    #     fp = 0
    #     fn = 0
    #     threshold = 0.5
    #     for i in range(len(predicts)):
    #         if predicts[i] >= threshold:
    #             if y_test[i] == 0.:
    #                 fp += 1
    #             else:
    #                 tp += 1
    #         else:
    #             if y_test[i] == 0.:
    #                 tn += 1
    #             else:
    #                 fn += 1
    #     acc = (tp + tn) / (tp + tn + fp + fn)
    #     precision = tp / (tp + fp)
    #     recall = tp / (tp + fn)
    #     f1_measure = 2 * tp / (2 * tp + fp + fn)
    #     print(model_path)
    #     print('acc          :', acc)
    #     print('precision    :', precision)
    #     print('recall       :', recall)
    #     print('f1_measure   :', f1_measure)
    #     print()

    def show_models_metrics(self):
        types = ['y', 'b', 'g', 'r', 'm', 'black', 'sandybrown', 'deepskyblue', 'purple']
        plt.plot([0, 1], [0, 1], 'k--')
        for model_path in self.get_files_path('h5'):
            start = time.time()
            label = model_path.split('/')[-2]
            x_labels, y_labels, x_labels_adv, y_labels_adv = self.roc(f'{self.save_dir}/{label}/model.h5', label)
            x_adv_labels, y_adv_labels, x_adv_labels_adv, y_adv_labels_adv = self.roc(f'{self.save_adv_dir}/{label}/model.h5', label)
            x_pre_labels, y_pre_labels, x_pre_labels_adv, y_pre_labels_adv = self.roc(f'{self.save_pre_adv_dir}/{label}/model.h5', label)
            # self.metric_infos(model_path)
            # plt.plot(x_labels, y_labels, types.pop(), label='normal_sample')
            plt.plot(x_labels_adv, y_labels_adv, types.pop(), label='nromal_model')
            plt.plot(x_adv_labels_adv, y_adv_labels_adv, types.pop(), label='adv_model')
            plt.plot(x_pre_labels_adv, y_pre_labels_adv, types.pop(), label='pre_adv_mdoel')
            plt.title(model_path.split('/')[-2])
            plt.xlabel('FPR')
            plt.ylabel('TPR')
            end = time.time()
            print(f'cost time: {end-start}')
            print('--------------------------------------')
            plt.legend()
            plt.show()

    def run(self):
        self.show_models_metrics()
        # self.show_history()
        # self.show_history(acc=True)
        # self.show_metrics()

if __name__ == '__main__':
    ShowResults('history_more').run()


from elasticsearch import Elasticsearch
import json
import string

def strip_punctuation(s):
    """Remove special characters and digits
    from a string.
    """
    sub1 = ''.join(c for c in s if c not in string.punctuation )
    return ''.join(c for c in sub1 if not c.isdigit())

# Set-up and define elasticsearch
es = Elasticsearch([{'host': 'localhost', 'port': 9200}])

# Get the query through system input
query = input()

def getAnswers(query):
    """Process a query in several steps and layers.
    Return a question to the user.
    """
    # An array with all unnecessary terms to be removed
    # from a query/answer
    stupidwords = ["och", "det", "att", "i", "en",
            "jag", "hon", "som", "han", "på", "den", "med", "var", "sig",
            "för", "så", "till", "är", "men", "ett", "om", "hade", "de", "av",
            "icke", "mig", "du", "henne", "då", "sin", "nu", "har", "inte",
            "hans", "honom", "skulle", "hennes", "där", "min", "man", "ej",
            "vid", "kunde", "något", "från", "ut", "när", "efter", "upp", "vi",
            "dem", "vara", "vad", "över", "än", "dig", "kan", "sina", "här",
            "ha", "mot", "alla", "under", "någon", "eller", "allt", "mycket",
            "sedan", "ju", "denna", "själv", "detta", "åt", "utan", "varit",
            "hur", "ingen", "mitt", "ni", "bli", "blev", "oss", "din", "dessa",
            "några", "deras", "blir", "mina", "samma", "vilken", "er", "sådan",
            "vår", "blivit", "dess", "inom", "mellan", "sådant", "varför",
            "varje", "vilka", "ditt", "vem", "vilket", "sitta", "sådana",
            "vart", "dina", "vars", "vårt", "våra", "ert", "era", "vilkas",
            "hej","hallå","tja","tjena","tjenare","dvs","osv","etc","","inkl","få","komma",
            "kommer"]

    # --- Pre-process the original query ---
    query = strip_punctuation(query)
    query = query.lower().split(" ")
    query = [word for word in query if word not in stupidwords]
    query = ' '.join(query)
    print(query)
    # -----------------------------

    # --- First search and post-search processing ---
    result = es.search(index="familjeliv", body={"query": {"match": {'question':query}}})
    interest = result['hits']['hits']
    newquery = query
    # Merge original query with at most top 5 questions
    # returned by elasticsearch
    for i in range(len(interest) if len(interest) < 5 else 5):
        score = interest[i]['_score']
        if(score > 1): #Filter-out terribly bad results
            newquery = newquery + ' '+interest[i]['_source']['question']
    # -----------------------------------------------

    # --- Pre-process the extended query ---
    newquery = strip_punctuation(newquery)
    newquery = newquery.lower().split(' ')
    finalresult = [word for word in newquery if word not in stupidwords]
    finalresult = ' '.join(finalresult)
    # --------------------------------------

    # --- Second search and extraction ---
    result1 = es.search(index="familjeliv", body={"query": {"match": {'question': finalresult}}})
    interest = result1['hits']['hits']
    #newquery = query
    final_answers = interest[0]['_source']['answers']
    # ------------------------------------

    # --- Create weights-system for terms ---
    finalresultarray = finalresult.split(" ") # Split extended query
    minHaspmap = {}
    for word in finalresultarray:
        if word in minHaspmap:
            minHaspmap[word] = minHaspmap[word] +1
        else:
            minHaspmap[word] = 1

    for word in query.lower().split(' '): #Split original query
        if word not in stupidwords:
            minHaspmap[word] = minHaspmap[word] + 1000
    # --------------------------------------

    # --- Answer processing ---
    score = -1
    ans = ""
    for answer in final_answers:
        answerlist = answer.lower().split(' ')
        tempscore = 0
        relevantwords = 0
        for word in answerlist:
            if word in minHaspmap:
                tempscore = tempscore + minHaspmap[word]
                relevantwords = relevantwords + 1
            if "?" in word: # Punish question marks
                tempscore = -999999
        tempscore = tempscore/len(answerlist) # Normalize score
        if score < tempscore: # Keep track of best answer
            ans = answer
            score = tempscore
    # --------------------------

    # --- Flagging and return ---
    finalans = {"answer":ans,"confidence":1 if score>=1 else 0} # Signal if good/bad answer
    return finalans
    # ---------------------------
out = getAnswers(query)
print(out)


from utils.constants import SECTOR_TYPES, EXIT_FLAGS, PLAYER_FLAGS
from utils.utils import GET_ALL_ROOM_EXITS, HAS_PLR_FLAG, MOB_IN_ROOM, GET_MOBS_IN_ROOM, CAN_SEE

SYMBOLS = {
	'you' : '{c[{b@{c]{n',
	SECTOR_TYPES['SECT_INSIDE'] : '{c[{w.{c]{n',
	SECTOR_TYPES['SECT_CITY'] : '{c[{WC{c]{n',
	SECTOR_TYPES['SECT_FIELD'] : '{c[{g,{c]{n',
	SECTOR_TYPES['SECT_FOREST']: '{c[{gY{c]{n',
	SECTOR_TYPES['SECT_HILLS']: '{c[{Mm{c]{n',
	SECTOR_TYPES['SECT_MOUNT']: '{c[{rM{c]{n',
	SECTOR_TYPES['SECT_SWIM']: '{c[{c~{c]{n',
	SECTOR_TYPES['SECT_SKY']: '{c[{W^{c]{n',
	SECTOR_TYPES['SECT_UNDRWATER']: '{c[{B~{c]{n',
	'HAS_MOB' : '{c[{m!{c]{n'

}

CLOSED = {
	'north' : "___",
	'south': '^^^',
	'east' : '|  ',
	'west' : '  |'
}

class Map(object):

	def __init__(self, ch, max_width=9, max_length=9):
		self.ch = ch
		self.max_width = max_width
		self.max_length = max_length
		self.worm_has_mapped = {}
		self.curX = None
		self.curY = None
		self.isClosed = [False, '']

		#{room: [posx, posy]}
		if self.check_grid():
			self.grid = self.create_grid()
			self.draw_room_on_map(ch.location, ((min(max_width, max_length) - 1) / 2))

	def update_pos(self, room, exit_name):
		#self.ch.msg("%r" % exit_name)
		self.curX, self.curY = self.worm_has_mapped[room][0], self.worm_has_mapped[room][1]
		if exit_name == 'east':
			self.curY += 1
		elif exit_name == 'west':
			self.curY -= 1
		elif exit_name == 'north':
			self.curX -= 1
		elif exit_name == 'south':
			self.curX += 1

	def draw_room_on_map(self, room, max_distance):
		closed_door = self.draw(room)
		if closed_door: max_distance = 0
		if max_distance == 0:
			#self.ch.msg("%r" % self.worm_has_mapped)
			#self.ch.msg("returning")
			return

		for Exit in GET_ALL_ROOM_EXITS(room):
			#self.ch.msg("%r leads to %r" % (Exit, Exit.destination))
			if self.has_drawn(Exit.destination): continue
			if Exit.name.lower() == 'up' or Exit.name.lower() == 'down': continue

			if not HAS_PLR_FLAG(self.ch, PLAYER_FLAGS['REDIT']):
				if EXIT_FLAGS['HIDDEN'] in Exit.db.exit_flags: continue

			if Exit.db.door_open is False:
				self.isClosed = [True, Exit.name.lower()]

			self.update_pos(room, Exit.name.lower())
			self.draw_room_on_map(Exit.destination, max_distance - 1)

	def draw(self, room):
		# first draw initial ch location on map
		if room == self.ch.location:
			self.start_loc_on_grid()
			self.worm_has_mapped[room] = [self.curX, self.curY]
			return
		self.worm_has_mapped[room] = [self.curX, self.curY]

		if self.curX == self.origX and self.curY == self.origY: return
		#self.ch.msg("%r" % self.worm_has_mapped[room])

		see_mobs = False
		if MOB_IN_ROOM(room):
			for mob in GET_MOBS_IN_ROOM(room):
				if CAN_SEE(self.ch, mob):
					see_mobs = True

		if see_mobs:
			self.grid[self.curX][self.curY] = SYMBOLS['HAS_MOB']
		else:
			self.grid[self.curX][self.curY] = SYMBOLS[room.db.sector_type]

		if self.isClosed[0]:
			self.grid[self.curX][self.curY] = CLOSED[self.isClosed[1]]
			self.isClosed = [False, ""]
			return True

	def start_loc_on_grid(self):
		x = self.median(self.max_width)
		y = self.median(self.max_length)
		self.grid[int(x)][int(y)] = SYMBOLS['you']
		self.curX = int(x); self.curY = int(y)
		self.origX = int(x); self.origY = int(y)
		#self.ch.msg("%r %r" % (self.curX, self.curY))

	def has_drawn(self, room):
		# check to see if drawn room is in list of drawn rooms
		# return true or false
		return True if room in self.worm_has_mapped.keys() else False

	def median(self, num):
		lst = sorted(range(0, num))
		n = len(lst)
		m = n -1
		return (lst[n/2] + lst[m/2]) / 2.0

	def check_grid(self):
		# pass check to see if maxw and maxl are prime numbers:
		if self.max_length % 2 != 0 or self.max_width % 2 != 0: return True
		else: return False

	def create_grid(self):
		board = []
		for row in range(self.max_width):
			board.append([])
			for column in range(self.max_length):
				board[row].append('   ')
		return board

	def show_map(self):
		map_string = ""
		for row in self.grid:
			map_string += "".join(row)
			map_string += "\n"

		return map_string
		#self.ch.msg("%s" % map_string)

class WorldMap(object):
	"""
	This will be the world map where players can traverse through tamriel
	"""
	pass


# coding=UTF-8

from django.forms import widgets
from django.utils.translation import ugettext_lazy as _

class Select2Widget(widgets.SelectMultiple):
    """
    Classe pour crée un widget qui doit se remplir en faisant des appels JSON
    (!) Pour que cela fonctionne, il faut passer l'URL en tant qu'attribut
        via 'data-select2-json',
    Exemple :
            known_languages = forms.CharField(
                label=a, max_length=100,
                widget=Select2Widget(attrs={
                    'title': a,
                    'placeholder': _(u'type here your known languages'),
                    'data-select2-json': reverse_lazy('json_url_langues'),
                    'class': 'form-control'}),
                error_messages=self.e)
    """
    def __init__(self, attrs=None):
        attrs = attrs if attrs else {}
        if not attrs.get('data-select2-json'):
            raise Exception(_('data-select2-json class needed'))
        super(Select2Widget, self).__init__(attrs)

    def validate(self, value):
        retour = super(Select2Widget, self).validate(value)
        return retour


import os

from django.conf import settings
from django.conf.urls import patterns, url
from django.contrib.admin import ModelAdmin
from django.contrib.admin.options import csrf_protect_m
from django.contrib.admin.util import unquote
from django.contrib.admin.views.main import IS_POPUP_VAR
from django.contrib.staticfiles.templatetags.staticfiles import static
from django.core.exceptions import PermissionDenied, ViewDoesNotExist
from django.core.urlresolvers import reverse
from django.db import transaction
from django.http import Http404, HttpResponse, HttpResponseRedirect
from django.shortcuts import get_object_or_404, render_to_response
from django.utils.translation import ugettext, ugettext_lazy as _

from media_tree import settings as app_settings
from media_tree.admin.actions import core_actions, maintenance_actions
from media_tree.admin.actions.utils import execute_empty_queryset_action
from media_tree.admin.utils import (set_current_request,
                                    get_request_attr, set_request_attr)
from media_tree.admin.views.change_list import SimpleFileNodeChangeList

from ..forms import SimpleFileForm
from ..models import FileNode

def mt_static(url):
    return static(app_settings.MEDIA_TREE_STATIC_SUBDIR + '/' + url)

class BaseFileNodeAdmin(ModelAdmin):
    change_list_template = 'admin/media_tree/filenode/base_change_list.html'

    list_display = app_settings.MEDIA_TREE_LIST_DISPLAY
    list_filter = app_settings.MEDIA_TREE_LIST_FILTER
    #list_display_links = app_settings.MEDIA_TREE_LIST_DISPLAY_LINKS
    search_fields = app_settings.MEDIA_TREE_SEARCH_FIELDS
    ordering = app_settings.MEDIA_TREE_ORDERING_DEFAULT

    _registered_actions = []

    class Media:
        js = [mt_static('lib/jquery/jquery-1.7.1.min.js'),
              mt_static('lib/jquery/jquery.ui.js'),
              mt_static('lib/jquery/jquery.cookie.js'),

              mt_static('lib/fileuploader.js'),
              mt_static('js/admin_enhancements.js'),
              mt_static('js/django_admin_fileuploader.js')]
        css = {'all': (mt_static('css/swfupload.css'),
                       mt_static('css/ui.css'))}

    # URLs

    def get_urls(self):
        urls = super(BaseFileNodeAdmin, self).get_urls()
        info = self.model._meta.app_label, self.model._meta.model_name

        mt_static = os.path.join(settings.STATIC_ROOT,
                                 app_settings.MEDIA_TREE_STATIC_SUBDIR)

        url_patterns = patterns('',

            url(r'^jsi18n/',
                self.admin_site.admin_view(self.i18n_javascript),
                name='media_tree_jsi18n'),

            url(r'^upload/$',
                self.admin_site.admin_view(self.upload_file_view),
                name='%s_%s_upload' % info),

            # Since Flash Player enforces a same-domain policy, the upload
            # will break if static files are served from another domain. So
            # the built-in static file view is used for the uploader SWF:
            url(r'^static/swfupload\.swf$',
                'django.views.static.serve',
                {'document_root': mt_static,
                 'path': 'lib/swfupload/swfupload_fp10/swfupload.swf'},
                name='%s_%s_static_swfupload_swf' % info))
        url_patterns.extend(urls)
        return url_patterns

    # Actions

    @staticmethod
    def register_action(func, required_perms=None):
        BaseFileNodeAdmin._registered_actions.append({
            'action': func,
            'required_perms': required_perms})

    def get_actions(self, request):
        # In ModelAdmin.get_actions(), actions are disabled if the popup var
        # is present. Since BaseFileNodeAdmin always needs a checkbox, this
        # is circumvented here:

        is_popup_var = request.GET.get(IS_POPUP_VAR, None)
        if IS_POPUP_VAR in request.GET:
            request.GET = request.GET.copy()
            del request.GET[IS_POPUP_VAR]

        # get all actions from parent
        actions = super(BaseFileNodeAdmin, self).get_actions(request)

        # and restore popup var
        if is_popup_var:
            request.GET[IS_POPUP_VAR] = is_popup_var

        return actions

    # Misc getters

    def get_changelist(self, request, **kwargs):
        """ Returns the ChangeList class for use on the changelist page. """
        return SimpleFileNodeChangeList

    def get_form(self, request, *args, **kwargs):
        self.fields = self.form.Meta.fields
        return super(BaseFileNodeAdmin, self).get_form(
            request, *args, **kwargs)

    # Add view

    @csrf_protect_m
    @transaction.commit_on_success
    def add_view(self, request, form_url='', extra_context=None):
        if not extra_context:
            extra_context = {}
        extra_context.update({'breadcrumbs_title': _('Add')})

        response = super(BaseFileNodeAdmin, self).add_view(
            request, form_url, extra_context)
        return response

    # Change view

    def change_view(self, request, object_id, extra_context=None):
        try:
            object_id = str(object_id)
            node = get_object_or_404(FileNode, pk=unquote(object_id))
        except ValueError:
            raise Http404
        set_request_attr(request, 'save_node', node)
        if not extra_context:
            extra_context = {}
        extra_context.update({'node': node,})
        return super(BaseFileNodeAdmin, self).change_view(\
            request, object_id, extra_context=extra_context)

    # Changelist view

    def get_changelist_view_options(self, request):
        extra_context = {}

        if app_settings.MEDIA_TREE_SWFUPLOAD:
            app, model = \
                self.model._meta.app_label, self.model._meta.model_name

            extra_context.update({
                'file_types': app_settings.MEDIA_TREE_ALLOWED_FILE_TYPES,
                'file_size_limit': app_settings.MEDIA_TREE_FILE_SIZE_LIMIT,
                'swfupload_flash_url': reverse(
                    'admin:%s_%s_upload' % (app, model)),
                'swfupload_upload_url': reverse(
                    'admin:%s_%s_static_swfupload_swf' % (app, model))})

        return extra_context

    def changelist_view(self, request, extra_context=None):
        response = execute_empty_queryset_action(self, request)
        if response:
            return response

        set_current_request(request)

        if not extra_context:
            extra_context = {}
        extra_context.update(self.get_changelist_view_options(request))

        if request.GET.get(IS_POPUP_VAR, None):
            extra_context.update({'select_button': True})

        response = super(BaseFileNodeAdmin, self) \
            .changelist_view(request, extra_context)
        return response

    # Upload view

    @csrf_protect_m
    @transaction.commit_on_success
    def upload_file_view(self, request):
        """ The upload view is exempted from CSRF protection since SWFUpload
            cannot send cookies (i.e. it can only send cookie values as POST
            values, but that would render this check useless anyway). However,
            Flash Player should already be enforcing a same-domain policy. """

        try:
            if not self.has_add_permission(request):
                raise PermissionDenied

            FILE_PARAM_NAME = 'qqfile'
            self.init_parent_folder(request)

            if request.method == 'POST':

                if request.is_ajax() and request.GET.get(FILE_PARAM_NAME, None):
                    content_file = ContentFile(request.body)
                    uploaded_file = UploadedFile(
                        content_file, request.GET.get(FILE_PARAM_NAME), None,
                        content_file.size)
                    form = UploadForm(request.POST, {'file': uploaded_file})
                else:
                    form = UploadForm(request.POST, request.FILES)

                if form.is_valid():
                    node = FileNode(file=form.cleaned_data['file'],
                                    node_type=media_types.FILE)
                    parent_folder = self.get_parent_folder(request)
                    if not parent_folder.is_top_node():
                        node.parent = parent_folder
                    self.save_model(request, node, None, False)
                    # Respond with 'ok' for the client to verify that the
                    # upload was successful, since sometimes a failed request
                    # would not result in a HTTP error and look like a
                    # successful upload. For instance: When requesting the
                    # admin view without authentication, there is a redirect
                    # to the login form, which to SWFUpload looks like a
                    # successful upload request.
                    if request.is_ajax():
                        return HttpResponse(
                            '{"success": true}',
                            content_type="application/json")
                    else:
                        messages.info(
                            request,
                            _('Successfully uploaded file %s.') % node.name)
                        return HttpResponseRedirect(
                            reverse('admin:%s_%s_changelist' % (
                                self.model._meta.app_label,
                                self.model._meta.model_name)))
                else:
                    # invalid form data
                    if request.is_ajax():
                        return HttpResponse('{"error": "%s"}' % ' '.join(
                            [item for sublist in
                             form.errors.values() for item in sublist]),
                            content_type="application/json")

            # Form is rendered for troubleshooting SWFUpload. If this form
            # works, the problem is not server-side.
            if not settings.DEBUG:
                raise ViewDoesNotExist
            if request.method == 'GET':
                form = UploadForm()
            return render_to_response(
                'admin/media_tree/filenode/upload_form.html', {'form': form})

        except Exception as e:
            if request.is_ajax():
                return HttpResponse('{"error": "%s"}' % ugettext('Server Error'),
                    content_type="application/json")
            else:
                raise

    def i18n_javascript(self, request):
        """ Displays the i18n JavaScript that the Django admin requires.

            This takes into account the USE_I18N setting. If it's set to False,
            the generated JavaScript will be leaner and faster. """
        return javascript_catalog(request, packages=['media_tree'])

BaseFileNodeAdmin.register_action(core_actions.copy_selected)
BaseFileNodeAdmin.register_action(maintenance_actions.delete_orphaned_files,
                                  ('media_tree.manage_filenode',))
BaseFileNodeAdmin.register_action(maintenance_actions.clear_cache,
                                  ('media_tree.manage_filenode',))


from routes.admin import BranchBase
from methods import Shortages as Methods
from methods import Types
from methods import Marks

class Shortages(BranchBase):
    route_base = '/shortages'

    methods = Methods

    strings = {
        'plural' : 'faltantes',
        'singular' : 'faltante',
    }

    subtemplate = 'shortages'

    table_option = {
        'deletable' : True,
    }

    columns = [
        { 'title' : 'Id',           'field' : '_id',        'hidden' : True },
        { 'title' : 'Codigo',       'field' : 'code',       'type' : 'upper' },
        { 'title' : 'Nombre',       'field' : 'name',       'type' : 'title', 'expand' : True },
        { 'title' : 'Stock',        'field' : 'stock',      'type' : 'numeric' },
        { 'title' : 'Venta min',    'field' : 'sale_min',   'type' : 'numeric' },
        { 'title' : 'Stock min',    'field' : 'stock_min',  'type' : 'numeric' },
    ]

    permission = 'inventories'

    def get_filters(self):
        filters = [
            { 'title' : 'Activo', 'key' : 'active', 'options' : [
                { 'name' : 'si', 'value' : True },
                { 'name' : 'no', 'value' : False },
            ]},
            { 'title' : 'Visible', 'key' : 'visible', 'options' : [
                { 'name' : 'si', 'value' : True },
                { 'name' : 'no', 'value' : False },
            ]},
            { 'title' : 'Granel', 'key' : 'scale', 'options' : [
                { 'name' : 'si', 'value' : True },
                { 'name' : 'no', 'value' : False },
            ]},
            { 'title' : 'Tipo de producto', 'key' : 'type', 'options' : [
                { 'name' : type['name'], 'value' : type['_id'] } for type in Types().list(active=True)
            ] },
            { 'title' : 'Marca', 'key' : 'mark', 'options' : [
                { 'name' : mark['name'], 'value' : mark['_id'] } for mark in Marks().list(active=True)
            ] },
        ]

        return filters


from tree import RGBXmasTree
from colorzero import Color
import paho.mqtt.client as mqtt

bright=0.1

def on_connect(client, obj, flags, rc):
    print("Connected with result code "+str(rc))
    client.subscribe("cheerlightsRGB")

def on_message(client, obj, msg):
    colorValue = msg.payload.decode('UTF-8')
    print(colorValue)
    tree.color=Color(colorValue)

def on_log(client, obj, level, msg):
    print(msg)

tree = RGBXmasTree(brightness=bright)

client = mqtt.Client()
client.on_connect  = on_connect
client.on_message = on_message
#client.on_log = on_log

client.connect("mqtt.cheerlights.com", 1883, 60)

client.loop_forever()


import numpy as np
import sys

import PyQt5.QtWidgets as qw
import PyQt5.QtCore as qc
import PyQt5.QtGui as qg

class App(qw.QMainWindow):
    def __init__(self):
        super().__init__()
        self.title = 'Aufgabe 1b: Spline Kurven'
        self.setMinimumHeight(qw.QDesktopWidget().height() // 3)
        self.setMinimumWidth(qw.QDesktopWidget().width() // 3)
        self.init_ui()

    def init_ui(self):
        self.setWindowTitle('Aufgabe 1b: Spline Kurven')
        self.setCentralWidget(DrawWidget(self))
        self.show()

class DrawWidget(qw.QWidget):
    def __init__(self, parent):
        qw.QWidget.__init__(self, parent)
        self.main_window = parent  # Main Window
        self.pane = Pane(self)
        sizePolicy = qw.QSizePolicy(qw.QSizePolicy.Ignored, qw.QSizePolicy.Ignored)
        self.pane.setSizePolicy(sizePolicy)
        #self.pane.setScaledContents(True)

        hbox = qw.QHBoxLayout()
        vbox = qw.QVBoxLayout()

        b_new = qw.QPushButton('New path', self)
        b_new.setToolTip('New path')
        b_new.clicked.connect(self.on_click_new_path)

        b_clear = qw.QPushButton('Clear', self)
        b_clear.setToolTip('Clear the Graph')
        b_clear.clicked.connect(self.on_click_clear)

        b_clear_pane = qw.QPushButton('Undo', self)
        b_clear_pane.setToolTip('remove last change')
        b_clear_pane.clicked.connect(self.on_click_undo)

        """b_plot = qw.QPushButton('Plot', self)
        b_plot.setToolTip('Plot a Graph')
        b_plot.clicked.connect(self.on_click_plot)"""

        """self.cm_type = qw.QComboBox(self)
        self.cm_type.addItem("normal")
        self.cm_type.addItem("speed up")
        self.cm_type.addItem("slow down")
        self.cm_type.activated.connect(self.on_click_type)"""

        cm_style = qw.QComboBox(self)
        cm_style.setToolTip('choose a style')
        cm_style.addItem("Fusion")
        cm_style.addItem("Windows")
        cm_style.addItem("macintosh")
        #m_style.addItem("Plastique")
        #cm_style.addItem("Cleanlooks")
        cm_style.addItem("windowsvista")
        cm_style.activated[str].connect(self.style_choice)

        b_show_cv = qw.QCheckBox('Show CV', self)
        b_show_cv.clicked.connect(lambda: self.on_click_show_cv(b_show_cv.checkState()))
        b_show_cv.setChecked(True)

        vbox.addWidget(b_new)
        vbox.addWidget(b_clear)
        vbox.addWidget(b_clear_pane)
        #vbox.addWidget(b_plot)
        #vbox.addWidget(self.cm_type)
        vbox.addWidget(cm_style)
        vbox.addWidget(b_show_cv)

        vbox.addStretch(1)

        hbox.addWidget(self.pane)

        hack = qw.QWidget()
        hack.setLayout(vbox)
        hack.setFixedWidth(110)

        hbox.addWidget(hack)

        self.setLayout(hbox)

    def on_click_clear(self):
        self.pane.clear_all_surfaces()

    def on_click_plot(self):
        self.pane.plot()

    def on_click_new_path(self):
        if len(self.pane.current_cv) != 0:
            self.pane.paths.append(self.pane.current_path)
            self.pane.current_path = qg.QPainterPath()

            self.pane.cvs.append(self.pane.current_cv)
            self.pane.current_cv = np.array([]).reshape(0, 2)

    def on_click_undo(self):
        self.pane.undo()

    def on_click_show_cv(self, value):
        self.pane.show_cv(value)

    def style_choice(self, text):
        qw.QApplication.setStyle(qw.QStyleFactory.create(text))

    def on_click_type(self):
        text = self.cm_type.currentText()
        self.pane.current_path.changed_style(text)
        self.pane.plot()

    def on_click_hight(self, text):
        self.pane.hight = int(text)
        self.pane.resolution_of_surfases()

    def on_click_width(self, text):
        self.pane.width = int(text)
        self.pane.resolution_of_surfases()

    def resizeEvent(self, event):
        self.pane.resolution_of_surfases()

class Pane(qw.QLabel):
    """ Paint surface class """
    def __init__(self, parent):
        """ creates Pane """
        super().__init__()
        self.parent = parent  # App - Main window
        main_window = parent.main_window
        self.style = "normal"
        self.color = qc.Qt.red
        self.thickness = 3
        self.grid = False

        # background for black background with lines of grid/crystal if activated
        self.ebene_pane = qg.QPixmap(self.width(), self.height())
        self.ebene_cv = qg.QPixmap(self.width(), self.height())
        self.ebene_schlitten = qg.QPixmap(self.width(), self.height())
        self.ebene_total = qg.QPixmap(self.width(), self.height())

        self.fill_all_default()     # cleans all surfaces
        self.set_all_painters()     # create all painters

        self.paths = []
        self.cvs = []
        self.current_path = qg.QPainterPath()

        self.painter_cv.setPen(qg.QPen(self.color, 3, qc.Qt.SolidLine))
        self.painter_cv.drawPath(self.current_path)

        self.current_cv = np.array([]).reshape(0, 2)
        self.showCV = True

        self.update()

    def resolution_of_surfases(self):
        self.end_all_painters()

        self.ebene_pane = qg.QPixmap(self.width(), self.height())
        self.ebene_cv = qg.QPixmap(self.width(), self.height())
        self.ebene_schlitten = qg.QPixmap(self.width(), self.height())
        self.ebene_total = qg.QPixmap(self.width(), self.height())

        #print(self.width(), self.h)
        self.fill_all_default()

        self.set_all_painters()

        self.update()
        self.plot()

    def fill_all_default(self):
        self.ebene_pane.fill(qg.QColor(0, 0, 0))
        self.ebene_cv.fill(qg.QColor(0, 0, 0, 0))
        self.ebene_schlitten.fill(qg.QColor(0, 0, 0, 0))
        self.ebene_total.fill(qg.QColor(0, 0, 0, 0))

    def set_all_painters(self):
        self.painter_cv = qg.QPainter(self.ebene_cv)
        self.painter_cv.setRenderHint(qg.QPainter.Antialiasing, True)
        self.painter_pane = qg.QPainter(self.ebene_pane)
        self.painter_pane.setRenderHint(qg.QPainter.Antialiasing, True)
        self.painter_schlitten = qg.QPainter(self.ebene_schlitten)
        self.painter_total = qg.QPainter(self.ebene_total)

    def end_all_painters(self):
        self.painter_cv.end()
        self.painter_pane.end()
        self.painter_schlitten.end()
        self.painter_total.end()

    def clear_all_surfaces(self):
        self.end_all_painters()
        self.fill_all_default()
        self.set_all_painters()

        self.paths = []
        self.current_path = qg.QPainterPath()

        self.cvs = []
        self.current_cv = np.array([]).reshape(0, 2)
        self.update()

    def undo(self):
        if len(self.current_cv) == 0:
            if len(self.cvs) != 0:
                self.current_cv = self.cvs[-1]
                self.cvs = self.cvs[:-1]
                self.paths = self.paths[:-1]

        if len(self.current_cv) != 0:
            self.current_cv = self.current_cv[:-1]
            if len(self.current_cv) != 0:
                self.current_path = qg.QPainterPath()
                self.current_path.moveTo(self.current_cv[0][0],self.current_cv[0][1])
                for i in range(len(self.current_cv)):
                    self.current_path.lineTo(self.current_cv[i][0], self.current_cv[i][1])
        self.update()
        self.plot()

    def show_cv(self, value):
        self.showCV = value
        self.update()

    def update(self):
        """ draws control points (vertexes) and Paths """
        """for i in range(self.k):
            if len(self.paths) == 0:
                return None
            if len(self.paths[i]) != 0:
                path = self.paths[i][-1][0]
                thickness = self.paths[i][-1][1]
                self.painter_pane.setPen(qg.QPen(self.color, thickness, qc.Qt.SolidLine))
                self.painter_pane.drawPath(path)"""

        """ draws control points and paths """
        self.ebene_cv.fill(qg.QColor(0, 0, 0, 0))

        if self.showCV:
            path = self.current_path
            color = qc.Qt.blue
            thickness = 1
            self.painter_cv.setPen(qg.QPen(color, thickness, qc.Qt.SolidLine))
            # draw actual path
            self.painter_cv.drawPath(path)
            # draw old paths
            for i in range(len(self.paths)):
                path = self.paths[i]
                self.painter_cv.drawPath(path)

            # draw old paths
            for j in range(len(self.cvs)):
                for i in range(self.cvs[j].shape[0]):
                    self.painter_cv.setPen(qg.QPen(color, 6, qc.Qt.SolidLine))
                    self.painter_cv.drawPoint(self.cvs[j][i][0], self.cvs[j][i][1])
            # draw actual path
            for i in range(self.current_cv.shape[0]):
                self.painter_cv.setPen(qg.QPen(color, 6, qc.Qt.SolidLine))
                self.painter_cv.drawPoint(self.current_cv[i][0],self.current_cv[i][1])

        self.ebene_total.fill(qg.QColor(0, 0, 0, 0))
        self.painter_total.drawPixmap(0, 0, self.ebene_pane)
        self.painter_total.drawPixmap(0, 0, self.ebene_cv)
        self.painter_total.drawPixmap(0, 0, self.ebene_schlitten)
        self.setPixmap(self.ebene_total)
        #print("drawing all")

    def plot(self):
        self.painter_pane.end()
        self.ebene_pane.fill(qg.QColor(0, 0, 0))
        self.painter_pane = qg.QPainter(self.ebene_pane)
        self.painter_pane.setRenderHint(qg.QPainter.Antialiasing, True)

        def p_1(t):
            return 1 / 6 * t ** 3

        def p_2(t):
            return 1 / 6 * (1 + 3 * t + 3 * t ** 2 - 3 * t ** 3)

        def p_3(t):
            return 1 / 6 * (4 - 6 * t ** 2 + 3 * t ** 3)

        def p_4(t):
            return 1 / 6 * (1 - 3 * t + 3 * t ** 2 - t ** 3)

        def b(t):
            if t <= 0:
                return 0
            elif t <= 1:
                return p_1(t)
            elif t <= 2:
                return p_2(t - 1)
            elif t <= 3:
                return p_3(t - 2)
            elif t <= 4:
                return p_4(t - 3)
            else:
                return 0

        def b_sp(t, i):
            return b(t - i)

        def change_color(color):
            if color%2 == 1:
                return qc.Qt.green
            else:
                return qc.Qt.darkCyan

        def test(x,y,color):
            path = qg.QPainterPath()

            break_j = 0
            farbe = change_color(color[0])
            self.painter_pane.setPen(qg.QPen(farbe, 3, qc.Qt.SolidLine))
            path.moveTo(x[0], y[0])
            for j in range(len(x)):
                path.lineTo(x[j], y[j])
                if j + 1 != len(color):
                    if color[j] != color[j + 1]:
                        break_j = j+1
                        break
            self.painter_pane.drawPath(path)
            if  break_j != 0:
                #print(f"len(x):{len(x)}, len(color):{len(color)}, break_j:{break_j}")
                return test(x[break_j:], y[break_j:], color[break_j:])

        def spline(cv):
            """
            :param cv: Control vertexes
            :return: x and y array for spline-curve
            """
            k = len(cv)  # Amount of CV
            if k < 4:
                print("insufficient control points")
                return False

            t_min, t_max = 1, k - 2  # Axe for shifted basis functions
            x = []
            y = []
            color = []
            color_temp = 0
            color_temp_v2 = 1
            t = t_min
            while t <= t_max:
                subsum_x = 0
                subsum_y = 0
                for i in range(0, k):
                    b = b_sp(t, i - 2)
                    # if b > 0.66:
                    #    color_temp = i
                    subsum_x += cv[i][0] * b
                    #print(f"P_x:{cv[i][0]} b_sp({t},{i}):{b}")
                    subsum_y += cv[i][1] * b
                if color_temp_v2 + 1 <= t:
                    color_temp_v2 += 1
                x.append(subsum_x)
                y.append(subsum_y)
                color.append(color_temp_v2)

                t += 0.01

            #print("x:", x)
            #print("y:", y)
            #print("color: ", color)
            return x, y, color

        def bspline(cv, n=100, degree=3):
            """ Calculate n samples on a bspline
                cv :      Array of control vertices
                n  :      Number of samples to return
                degree:   Curve degree
            """
            cv = np.asarray(cv)
            count = len(cv)

            degree = np.clip(degree,1,count-1)

            # Calculate knot vector
            kv = None
            kv = np.clip(np.arange(count+degree+1)-degree,0,count-degree)

            # Calculate query range
            u = np.linspace(False,(count-degree),n)

            # Calculate result
            #return np.array(si.splev(u, (kv,cv.T,degree))).T

        self.painter_pane.setPen(qg.QPen(qc.Qt.red, 3, qc.Qt.SolidLine))

        for i in range(len(self.cvs)):
            #p = bspline(self.cvs[i], n=len(self.cvs[i])*10)
            #x, y = p.T
            temp = spline(self.cvs[i])
            if temp:
                x,y,color = temp[0], temp[1], temp[2]
                test(x,y,color)
                """
                path = qg.QPainterPath()

                path.moveTo(x[0],y[0])
                for j in range(len(x)):
                    path.lineTo(x[j],y[j])

                self.painter_pane.drawPath(path)"""

        # actual path, if it is not empty!
        if len(self.current_cv) != 0:
            #p = bspline(self.current_cv, n=len(self.current_cv)*10) # n - amount of interpolated points
            #x, y = p.T
            temp = spline(self.current_cv)
            if temp:
                x,y, color = temp[0], temp[1], temp[2]
                #if len(x) > 10:
                    #x, y = x[5:], y[5:]
                    #x, y = x[:-5], y[:-5]
                #print(x)
                test(x,y, color)
                """
                path = qg.QPainterPath()

                path.moveTo(x[0], y[0])
                for j in range(len(x)):
                    path.lineTo(x[j], y[j])
                    if j+1 != len(x):
                        if color[j] != color[j+1]:

                self.painter_pane.drawPath(path)"""

        self.update()

        #print(self.current_cv)
        #p = bspline(self.test_cv, n=100)
        #x, y = p.T
        #print(x,y)

    def mousePressEvent(self, event):
        #print("click")
        x = event.pos().x()
        y = event.pos().y()
        if self.current_path.isEmpty():
            self.current_path.moveTo(x, y)
            self.current_path.lineTo(x+0.1, y+0.1)
        self.current_path.lineTo(x, y)

        self.current_cv = np.r_[self.current_cv, [[x, y]]]
        #print(self.current_cv, self.current_cv[:, 0] )

        self.update()
        self.plot()

if __name__ == '__main__':
    app = qw.QApplication(sys.argv)
    ex = App()
    sys.exit(app.exec_())


import argparse
import csv
import itertools
import tqdm
from pathlib import Path
from typing import *

import cv2
import pandas as pd

def get_images(path: Path) -> List[Path]:
    files_iter = itertools.chain(
        path.rglob("*.jpg"),
        path.rglob("*.png"),
        path.rglob("*.jpeg"),
        path.rglob("*.JPG"),
        path.rglob("*.bmp"),
    )
    files_list = list(files_iter)
    return files_list

def load_annotation(p: Path):
    with open(p.as_posix(), 'r') as f:
        reader = csv.reader(f)
        for line in reader:
            # strip BOM. \ufeff for python3,  \xef\xbb\bf for python2
            line = [i.strip('\ufeff').strip('\xef\xbb\xbf') for i in line]

            x1, y1, x2, y2, x3, y3, x4, y4 = list(map(int, line[:8]))
            yield x1, y1, x2, y2, x3, y3, x4, y4

def parse_dataset(path: str):
    imgs = get_images(Path(path))
    data = []
    for img_fn in tqdm.tqdm(imgs):
        img = cv2.imread(img_fn.as_posix())
        annotations = load_annotation(img_fn.with_suffix(".txt"))
        for x1, y1, x2, y2, x3, y3, x4, y4 in annotations:
            line = [
                str(img_fn.as_posix()),
                img.shape[1],
                img.shape[0],
                "sign",
                min(x1, x2, x3, x4),
                min(y1, y2, y3, y4),
                max(x1, x2, x3, x4),
                max(y1, y2, y3, y4),
            ]
            data.append(line)
    path = Path(path)
    dataframe = pd.DataFrame(data=data,
                             columns=['filename', 'width', 'height', 'class', 'xmin', 'ymin', 'xmax', 'ymax'])
    dataframe.to_csv(path.with_suffix(".csv"), index=False)

if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument("--dataset")
    args = parser.parse_args()
    parse_dataset(args.dataset)


#!/usr/local/bin/python3
# -*- coding: UTF-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import sys
import threading
import random
from datetime import datetime

import numpy as np
from six.moves import xrange
import xml.etree.ElementTree as xml_parser

import tensorflow as tf

# 设置日志级别。
tf.logging.set_verbosity(tf.logging.INFO)

def convert2_int64_feature(value):
    """ 将 int64 类型值，包装成训练样本的格式 """
    if not isinstance(value, list):
        value = [value]
    return tf.train.Feature(int64_list=tf.train.Int64List(value=value))

def convert2_float_feature(value):
    """ 将 float 类型值，包装成训练样本的格式 """
    if not isinstance(value, list):
        value = [value]
    return tf.train.Feature(float_list=tf.train.FloatList(value=value))

def convert2_bytes_feature(value):
    """ 将 bytes 类型值，包装成训练样本的格式 """
    return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))

def convert2_train_example(filename, image_buffer, label, synset, human, bbox,
                           height, width):
    """
    将一个图像转换成一个训练样本。

    @param filenames: 字符串列表。每个字符串代表一个图片文件。
    @param image_buffer: 字符串。JPEG编码的RGB图像。
    @param labels: 整数列表。每个整数代表一个图片的类别。
    @param synsets: 字符串列表。每个字符串代表一个同义词标识（WordNet ID）。
    @param humans: 字符串列表。每个字符串代表一个可读的类别名称。
    @param bbox: 边界框列表，每个图像有0个或者多个边界框，每个边界框都由
      [xmin, ymin, xmax, ymax]组成
    @param height: 整型, 图像的高度（像素）
    @param width: 整型, 图像的宽度（像素）

    @Returns: proto格式的样本数据
    """

    xmin = []
    ymin = []
    xmax = []
    ymax = []
    for b in bbox:
        [l.append(point) for l, point in zip([xmin, ymin, xmax, ymax], b)]

    # 如果没有读取到对象边界框，那么，返回None
    if len(xmin) == 0:
        return None

    colorspace = 'RGB'
    channels = 3
    image_format = 'JPG'

    example = tf.train.Example(features=tf.train.Features(feature={
        'image/height': convert2_int64_feature(height),
        'image/width': convert2_int64_feature(width),
        'image/colorspace': convert2_bytes_feature(colorspace.encode()),
        'image/channels': convert2_int64_feature(channels),
        'image/class/label': convert2_int64_feature(label),
        'image/class/synset': convert2_bytes_feature(synset.encode()),
        'image/class/text': convert2_bytes_feature(human.encode()),

        # 一个图片中，包含多个对象边界框时，是将所有的 xmin 放在一个列表中，同理，
        # 将所有的 xmax、ymin、ymax 也放在一个列表内
        'image/object/bbox/xmin': convert2_int64_feature(xmin),
        'image/object/bbox/xmax': convert2_int64_feature(xmax),
        'image/object/bbox/ymin': convert2_int64_feature(ymin),
        'image/object/bbox/ymax': convert2_int64_feature(ymax),

        # 每个对象边界框，都对应一个 label，
        'image/object/bbox/label': convert2_int64_feature([label] * len(xmin)),
        'image/format': convert2_bytes_feature(image_format.encode()),
        'image/filename': convert2_bytes_feature(
            os.path.basename(filename).encode()),
        'image/encoded': convert2_bytes_feature(image_buffer)}
    ))
    return example

class JpegImageCoder(object):
    """ ImageNet数据读取时解码用工具类 """

    def __init__(self):
        # 为图像编码单独创建一个会话
        self._sess = tf.Session()

        # 将PNG图像转换成JPEG格式图片的功能函数
        self._png_data = tf.placeholder(dtype=tf.string)
        image = tf.image.decode_png(self._png_data, channels=3)
        self._png_to_jpeg = tf.image.encode_jpeg(
            image, format='rgb', quality=100)

        # 将 CMYK 图像转换成 RGB JPEG 格式图片的功能函数
        self._cmyk_data = tf.placeholder(dtype=tf.string)
        image = tf.image.decode_jpeg(self._cmyk_data, channels=0)
        self._cmyk_to_rgb = tf.image.encode_jpeg(
            image, format='rgb', quality=100)

        # 初始化解码 RGB JPEG 图像函数
        self._decode_jpeg_data = tf.placeholder(dtype=tf.string)
        self._decode_jpeg = tf.image.decode_jpeg(
            self._decode_jpeg_data,
            # 有时候，JPEG图片没有完全下载，无法完美解析，所以，
            # 让 decode_jpeg 容忍一定程度的异常和错误，提高适应性
            # 代价是，会降低最终的模型识别准确率降低。
            try_recover_truncated=True,
            acceptable_fraction=0.75,
            channels=3)

    # 将 PNG 格式转换成 JPEG 格式
    def png_to_jpeg(self, image_data):
        return self._sess.run(self._png_to_jpeg,
                              feed_dict={self._png_data: image_data})

    # 将 CMYK 格式转换成 JPEG 格式
    def cmyk_to_rgb(self, image_data):
        return self._sess.run(self._cmyk_to_rgb,
                              feed_dict={self._cmyk_data: image_data})

    # 将 JPEG 图像文件编码成二进制格式
    def decode_jpeg(self, image_data):
        image = self._sess.run(self._decode_jpeg,
                               feed_dict={self._decode_jpeg_data: image_data})
        assert len(image.shape) == 3
        assert image.shape[2] == 3
        return image

def _is_png(filename):
    """
    检测一个图像文件是否是 PNG 格式。

    @param filenames: 字符串列表。每个字符串代表一个图片文件。

    @Returns: 布尔值，表明该图标是否是 PNG 格式的图像。
    """

    # 文件列表来源自:
    # https://groups.google.com/forum/embed/?place=forum/torch7#!topic/torch7/fOSTXHIESSU
    # 请注意：文件名后缀的大小写。'jpeg' vs 'jpg'
    return 'n02105855_2933.jpg' in filename

def _is_cmyk(filename):
    """
    检测图像文件是否使用一个CMYK 色彩看空间的 JPEG 图像。

    @param filenames: 字符串列表。每个字符串代表一个图片文件。

    @Returns: 布尔值，表明该图标是否是 采用 CMYK 色彩看空间的 JPEG 图像编码。
    """

    # 文件列表来源自:
    # https://github.com/cytsai/ilsvrc-cmyk-image-list
    # 请注意：文件名后缀的大小写。
    blacklist = ['n01739381_1309.jpg', 'n02077923_14822.jpg',
                 'n02447366_23489.jpg', 'n02492035_15739.jpg',
                 'n02747177_10752.jpg', 'n03018349_4028.jpg',
                 'n03062245_4620.jpg', 'n03347037_9675.jpg',
                 'n03467068_12171.jpg', 'n03529860_11437.jpg',
                 'n03544143_17228.jpg', 'n03633091_5218.jpg',
                 'n03710637_5125.jpg', 'n03961711_5286.jpg',
                 'n04033995_2932.jpg', 'n04258138_17003.jpg',
                 'n04264628_27969.jpg', 'n04336792_7448.jpg',
                 'n04371774_5854.jpg', 'n04596742_4225.jpg',
                 'n07583066_647.jpg', 'n13037406_4650.jpg']
    return os.path.basename(filename) in blacklist

def process_single_image(filename, coder):
    """
    读取一个单一的图像文件，转换成 JPG 格式。

    @param filenames: 字符串列表。每个字符串代表一个图片文件。
    @param coder: ImageCoder的实例，用于读取图片、或者对图片进行解码。

    @Returns:
      image_buffer: 字符串。JPEG编码的RGB图像。
      height: 整型, 图像的高度（像素）。
      width: 整型, 图像的宽度（像素）。
    """
    # 读取图像文件
    image_data = tf.gfile.FastGFile(filename, 'rb').read()

    # 判断图像是否是 PNG 格式
    if _is_png(filename):
        # 1 将图像从 PNG 格式转换成 JPEG 格式
        print('将图像从 PNG 转换成 JPEG 格式 %s' % filename)
        image_data = coder.png_to_jpeg(image_data)

    # 判断图像是否是 CMYK 格式
    elif _is_cmyk(filename):
        # 22 将图像从 CMYK 格式转换成 JPEG 格式
        print('将图像从 CMYK 转换成 RGB 格式 %s' % filename)
        image_data = coder.cmyk_to_rgb(image_data)

    # 按照 RGB 格式，对图像数据进行解码
    image = coder.decode_jpeg(image_data)

    assert len(image.shape) == 3
    height = image.shape[0]
    width = image.shape[1]
    assert image.shape[2] == 3

    return image_data, height, width

def process_batch_images(coder, thread_index, ranges, name, filenames,
                         synsets, labels, humans, bboxes, num_shards):
    """
    处理图片并且将图片转换成为 TFRecord 的单个线程。

    @param coder: ImageCoder的实例，用于读取图片、或者对图片进行解码。
    @param thread_index: 整型，一个批次中的索引值，取值范围[0, len(ranges)).
    @param ranges: 两个整型数值，表示并行处理过程中一个批次的范围。
    @param name: 字符串，唯一标识数据集。
    @param filenames: 字符串列表。每个字符串代表一个图片文件。

    @param synsets: 字符串列表。每个字符串代表一个同义词标识（WordNet ID）。
    @param labels: 整数列表。每个整数代表一个图片的类别。
    @param humans: 字符串列表。每个字符串代表一个可读的类别名称。
    @param bboxes: 边界框列表，每个图像有0个或者多个边界框。
    @param num_shards: 数据分片，并行处理时每个批次处理的图片个数。
    """

    # 每个线程处理 N 份数据， 其中，N = int(num_shards / num_threads).
    # 例如，如果每个分片含128个图片（num_shards = 128），采用两个线程并行处理
    # （um_threads = 2），那么，第一个线程处理的分片范围 [0, 64)
    num_threads = len(ranges)
    num_shards_per_batch = int(num_shards / num_threads)

    # 每个分片的范围，在每个线程分得的任务内，再次分片
    # 对于 ImageNet 来说，训练集包含大约100万个样本图片，如果采用 10 个线程并行处理
    # 那么，每个线程大约分得10多万个图片数据，在线程内，需要将本线程获得的任务再次分片
    shard_ranges = np.linspace(ranges[thread_index][0],
                               ranges[thread_index][1],
                               num_shards_per_batch + 1).astype(int)
    # 每个线程处理的图片数量
    num_files_in_thread = ranges[thread_index][1] - ranges[thread_index][0]

    counter = 0

    # s 是图片文件的索引
    for idx in xrange(num_shards_per_batch):
        # 生成一个数据分片，将一个数据分片的中包含的图片数据、标签数据、对象边界框转换成TFRecord
        # 然后，保存到一个样本文件中，例如，'train-00002-of-00010'
        shard = thread_index * num_shards_per_batch + idx
        output_filename = '%s-%.5d-of-%.5d' % (name, shard, num_shards)

        # 根据不同的数据集名称，将结果文件保存到对应的文件夹
        output_file = os.path.join(data_dir, name, output_filename)

        # 将样本数据写入到对应的样本数据文件的句柄
        writer = tf.python_io.TFRecordWriter(output_file)

        shard_counter = 0
        files_in_shard = np.arange(
            shard_ranges[idx], shard_ranges[idx + 1], dtype=int)
        for i in files_in_shard:
            filename = filenames[i]
            label = labels[i]
            synset = synsets[i]
            human = humans[i]
            bbox = bboxes[i]

            # 读取图片的内容，一律转换成为 JPG 格式
            image_buffer, height, width = process_single_image(filename, coder)

            # 将图片转换成为训练样本，TFRecord
            example = convert2_train_example(
                filename, image_buffer, label, synset, human,
                bbox, height, width)

            if example is not None:
                # 写入 TFRecord 格式的训练样本中
                # 每个训练样本中包含 num_shards 个图片文件
                writer.write(example.SerializeToString())
                shard_counter += 1
                counter += 1

            # 每处理 1000 个图片文件，
            if not counter % 1000:
                msg = "{} [{}]: 正在处理第 {}/{} 个图像".format(
                    datetime.now(), thread_index, counter,
                    num_files_in_thread)
                print(msg)
                sys.stdout.flush()

            writer.close()

        print('%s [线程 %d]: 已经将 %d 个图片写入 %s' %
              (datetime.now(), thread_index, shard_counter, output_file))
        sys.stdout.flush()
        shard_counter = 0

    print('\n%s [线程 %d]: 已经将 %d 个图片写入 %d 数据分片中.' %
          (datetime.now(), thread_index, counter, num_files_in_thread))
    sys.stdout.flush()

def build_human_labels(synsets, human_readable_labels):
    """
    构建可读标签的列表。

    @param synsets: 字符串列表。每个字符串代表一个同义词标识（WordNet ID）。
    @param synset_to_human: 字典。从图片所属的类别到可读的类别映射，例如：
      'n02119022' --> 'red fox, Vulpes vulpes'

    @Returns: 与同义词一一对应的刻度标签列表。
    """
    humans = []
    for s in synsets:
        assert s in human_readable_labels, ('没有找到: %s的可读标签。' % s)
        humans.append(human_readable_labels[s])
    return humans

def build_bounding_boxes(filenames, image_to_bboxes):
    """
    给定一个图片，读取它所包含的对象边界框列表。

    @param filenames: 字符串列表。每个字符串代表一个图像的文件名称。
    @param image_to_bboxes: 字典，从图像文件名到对象边界框列表的映射，列表中
        可以包含0个或者多个对象边界框。

    @Returns:
      针对每一个图像的对象边界框列表，每一个图片都可以包含多个对象边界框。
    """
    num_image_bbox = 0
    bboxes = []
    for f in filenames:
        # 字典的 key， 本例中采用文件的 basename
        basename = os.path.basename(f)

        if basename in image_to_bboxes:
            bboxes.append(image_to_bboxes[basename])
            num_image_bbox += 1

        # 如果 image_to_bboxes 中没有该文件的对象边界框，那么，
        # 创建一个空的对象边界框列表（对应于该图像中没有对象边界框）
        # 或者 该图像中有对象，但是却没有为它创建对象边界框
        else:
            bboxes.append([])

    print('找到 %d 个图片的共计 %d 对象边界框' % (
        len(filenames), num_image_bbox))

    return bboxes

def process_imagenet_images(name, filenames, synsets, labels, humans,
                            bboxes, num_shards):
    """
    处理所有的图像，并且，将图像转换成为 TFRecord 格式，保存起来。

    @param name: 字符串。数据集的唯一标识。
    @param filenames: 字符串列表。每个字符串代表一个图像的文件名称。
    @param synsets: 字符串列表。每个字符串代表一个同义词标识（WordNet ID）。
    @param labels: 整数列表。每个整数代表一个图片的类别。
    @param humans: 字符串列表。每个字符串代表一个可读的类别名称。
    @param bboxes: 边界框列表，每个图像有0个或者多个边界框。
    @param num_shards: 并行处理时每个形成处理的图片个数.
    """
    assert len(filenames) == len(synsets)
    assert len(filenames) == len(labels)
    assert len(filenames) == len(humans)
    assert len(filenames) == len(bboxes)

    # 按照并行线程的数量，将图像数据分成多个分片
    spacing = np.linspace(
        0, len(filenames), num_threads + 1).astype(np.int)
    ranges = []
    threads = []

    # 计算每个分片的起止范围
    # 每个批次的起止范围 [ranges[i][0], ranges[i][1]].
    # 例如，总样本数量是130万个，线程数量是10个，那么，每个线程处13万个图片
    # 因为, 线程数量是10个，所以，len(ranges) = 10。
    # 对应的，ranges[0][0] = 0, ranges[0][1] = 130000。
    # 对应的，ranges[1][0] = 130001, ranges[1][1] = 160000。
    #           ……
    #        åranges[9][0] = 1170001, ranges[9][1] = 1300000
    for i in xrange(len(spacing) - 1):
        ranges.append([spacing[i], spacing[i+1]])

    # 加载一个线程，处理一个批次数据
    print('启动第 {}个线程，处理 {} 批次数据'.format(num_threads, ranges))
    sys.stdout.flush()

    # 启动线程同步机制，监控等到所有的线程执行完毕
    coord = tf.train.Coordinator()

    # 创建一个图像编码器
    coder = JpegImageCoder()

    threads = []
    for thread_index in xrange(len(ranges)):
        # 每个线程的参数
        args = (coder, thread_index, ranges, name, filenames,
                synsets, labels, humans, bboxes, num_shards)
        # 创建线程，指定参数
        t = threading.Thread(target=process_batch_images, args=args)
        # 启动线程
        t.start()
        threads.append(t)

    # 将线程加入到同步机制中，确保所有线程执行完毕
    coord.join(threads)

    print('%s: 执行完毕，已经将所有的，共 %d 个图片保存到数据集中。' %
          (datetime.now(), len(filenames)))
    sys.stdout.flush()

def process_imagenet(name, num_shards, synset_to_human):
    """
    处理 ImageNet 数据集。

    @param name: 字符串。数据集名称，如 训练集train、验证集validation。
    @param num_shards: 数据集分成几个分片来处理，每个分片处理多少个图片.
    @param synset_to_human: 字典。从图片所属的类别到可读的类别映射，例如：
      'n02119022' --> 'red fox, Vulpes vulpes'
    """
    filenames, synsets, labels = find_image_files(
        data_dir, labels_file)
    image_to_bboxes = read_annotation_file(filenames)
    if len(image_to_bboxes) > 0:
        humans = build_human_labels(synsets, synset_to_human)
        bboxes = build_bounding_boxes(filenames, image_to_bboxes)
        process_imagenet_images(name, filenames, synsets, labels,
                                humans, bboxes, num_shards)
    else:
        print("没有找到对象边界框！")

def find_image_files(data_dir, labels_file):
    """
    读取 ImageNet 图片数据集。

    @param data_dir: 字符串. ImageNet图片集所在的本地目录。
    图片文件名称的格式如下所示：
        ${data_dir}/images/n02084071/n02084071_7.JPG
        ${data_dir}/images/n02084071/n02084071_60.JPG
    其中，n02084071 代表图片所属的类别（同义词 ID）

    一个图片中会包含多个对象，对象所在的位置大小、类别数据，保存在如下XML文件中：
        ${data_dir}/annotation/n02084071/n02084071_7.xml
        ${data_dir}/annotation/n02084071/n02084071_60.xml
    其中，n02084071 代表图片所属的类别（同义词 ID）

    @param labels_file: 字符串。包含类别名称的文本文件，每一行代表一个类别，如：
        n02084071
        n04067472
        n04540053

    @Returns:
      filenames: 字符串列表; 每个字符串代表一个图片文件的名称.
      synsets: 字符串列表; 每个字符串是一个同义词唯一标识（WordNet ID）.
      labels: 整数列表; 代表图片所属的类别.
    """

    labels_file = os.path.join(data_dir, labels_file)
    challenge_synsets = [l.strip() for l in
                         tf.gfile.FastGFile(labels_file, 'r').readlines()]

    labels = []
    filenames = []
    synsets = []

    # Label索引，索引0保留
    label_index = 1

    # 构建图片文件名称列表和对应的labels列表
    for synset in challenge_synsets:
        jpeg_file_path = os.path.join(data_dir, 'images', synset)
        # 如果，${data_dir}/images/${synset} 文件夹不存在
        # 说明该同义词对应的图片都不存在，略过
        if not os.path.exists(jpeg_file_path):
            print("路径：{} 不存在".format(jpeg_file_path))
            continue

        # 找到指定路径下的所有“*.jpg”图片
        jpeg_file_path = os.path.join(jpeg_file_path, '*.jpg')
        matching_files = tf.gfile.Glob(jpeg_file_path)

        labels.extend([label_index] * len(matching_files))
        synsets.extend([synset] * len(matching_files))
        filenames.extend(matching_files)

        if not label_index % 100:
            print('共找到 {} 个类别 {} 个文件.'.format(
                label_index, len(challenge_synsets)))
        label_index += 1

    # 对图片进行乱序操作，让模型训练保证足够的多样性
    shuffled_index = list(range(len(filenames)))
    random.seed(12345)
    random.shuffle(shuffled_index)

    filenames = [filenames[i] for i in shuffled_index]
    synsets = [synsets[i] for i in shuffled_index]
    labels = [labels[i] for i in shuffled_index]

    print('在 {} 文件夹下，查找 {} 类别图片，共找到 {} 个图片文件。'.format
          (data_dir, len(challenge_synsets), len(filenames)))
    return filenames, synsets, labels

def read_synset_map(synset_to_human_file):
    """
    生成图片 类别ID 与 所属类别名称的映射。

    @synset_to_human_file: 字符串。包含图片类别ID 与 类别映射数据的文件.
    文件内容如下所示:
          n02119247    black fox
          n02119359    silver fox
          n02119477    red fox, Vulpes fulva

    每行包含一个映射关系，格式如：
    <同义词标识>\t<可读的类别名称>.

    @Returns:
      类别ID 到 类别名称的字典
    """

    # 读取文件内容
    synset_to_human_file = os.path.join(data_dir, synset_to_human_file)
    lines = tf.gfile.FastGFile(synset_to_human_file, 'r').readlines()
    # 要返回的 { 类别标识 -> 类别名称的 } 字典
    synset_to_human = {}
    for line in lines:
        if line:
            parts = line.strip().split('\t')
            assert len(parts) == 2
            synset = parts[0]
            human = parts[1]
            synset_to_human[synset] = human

    return synset_to_human

def read_annotation_file(image_files):
    """
    读取各个图片中包含所有对象的框，返回根据文件名称到框的映射。

    @param image_files: 字符串列表。图片文件名称列表。
    每个文件名称形如：n02084071_79.jpg，每个图片对应的有一个annotation文件
    位于： ${data_dir}/annotation/${synset}/${synset}_*.xml

    该XML文件中包含0个或者多个对象，所包含的对象大小为：

    <object>
      <name>n02084071</name>
      <pose>Unspecified</pose>
      <truncated>0</truncated>
      <difficult>0</difficult>
      <bndbox>
        <xmin>0</xmin>
        <ymin>102</ymin>
        <xmax>182</xmax>
        <ymax>387</ymax>
      </bndbox>
    </object>

    @Returns:
      图片到该图片所包含的对象框列表的字典
    """
    images_to_bboxes = {}
    num_bbox = 0
    num_image = 0
    for image_filename in image_files:
        image_basename = os.path.basename(image_filename)
        tmp = os.path.splitext(image_basename)[0]
        synset = tmp[0:tmp.index('_')]
        xml_file = "./data/imagenet/annotation/{}/{}.xml".format(
            synset, tmp)
        if not tf.gfile.Exists(xml_file):
            print("图片 {} 对应的 annotation： {} 文件不存在".format(
                image_basename, xml_file))
            continue

        if xml_file:
            # 解析XML文件
            xmldoc = xml_parser.parse(xml_file)

            # 读取所有的对象列表
            object_list = xmldoc.findall('object')

            # 读取该对象坐标的对标位置（对应的像素）
            for object_element in object_list:
                xmin = int(object_element.find('bndbox/xmin').text)
                ymin = int(object_element.find('bndbox/ymin').text)
                xmax = int(object_element.find('bndbox/xmax').text)
                ymax = int(object_element.find('bndbox/ymax').text)

            # 对象所在的边界框
            box = [xmin, ymin, xmax, ymax]

            # 以图片文件的basename为主键，是包含'.jpg'的后缀的
            if image_basename not in images_to_bboxes:
                images_to_bboxes[image_basename] = []
                num_image += 1

            # 图片到 边界框的映射（Map）
            images_to_bboxes[image_basename].append(box)
            num_bbox += 1

        print('针对图片文件{}, 读取到 {} 个对象'.format(
            image_basename, len(images_to_bboxes[image_basename])))

    return images_to_bboxes

def process_imagenet_data():
    """

    @Returns:
      图片到该图片所包含的对象框列表的字典
    """

    # Build a map from synset to human-readable label.
    synset_to_human = read_synset_map(synset_to_human_file)

    # 数据分片大小，测试集每个分片包括1024个样本
    train_shards = 1024
    # 验证集中，每个数据分片的大小为128个图片
    validation_shards = 128
    process_imagenet('train', train_shards,
                     synset_to_human)
    process_imagenet('validation',
                     validation_shards, synset_to_human)

# 相关参数
data_dir = './data/imagenet/'
train_dir = './data/imagenet/train/'
validation_dir = './data/imagenet/validation/'
os.makedirs(train_dir, exist_ok=True)
os.makedirs(validation_dir, exist_ok=True)

labels_file = 'synset_list.txt'
synset_to_human_file = 'synset_to_human.txt'

# ImageNet 包含图片大约1.3万个，需要并行处理，本例中采用4个线程变形处理
num_threads = 4

# ImageNet图像数据集文件，生成 TFRecord 格式的训练样本
process_imagenet_data()


#!/usr/bin/env python
#coding=utf8

import os.path
import sys

src_path = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))

if src_path not in sys.path:
    lib_path = os.path.join(src_path, 'lib')
    sys.path.append(src_path)
    sys.path.append(lib_path)

import tornado.httpserver
import tornado.ioloop
import tornado.options
import tornado.web

from bkmfc.config import configs, load_conf_file

class Application(tornado.web.Application):
    def __init__(self):
        #db = hqby.db.get_conn('fitter_sport')
        handlers = []
        handler_mods = [
            'auth',
            'user',
        ]
        for i in handler_mods:
            m = __import__('handler.' + i, fromlist=['url_spec'])
            handlers.extend(m.url_spec())

        settings = dict(
            template_path=os.path.join(os.path.dirname(__file__), 'templates'),
            static_path=os.path.join(os.path.dirname(__file__), 'static'),
            cookie_secret=configs['cookie_secret'],
            autoescape=None,
            debug=configs['debug'],
        )
        tornado.web.Application.__init__(self, handlers, **settings)

def main():
    http_server = tornado.httpserver.HTTPServer(Application(), xheaders=True)
    http_server.listen(configs['port'])
    try:
        tornado.ioloop.IOLoop.instance().start()
    except KeyboardInterrupt:
        pass

if __name__ == '__main__':
    load_conf_file(os.path.join(src_path, 'etc', 'db.conf'))
    load_conf_file(os.path.join(src_path, 'etc', 'mfc.conf'))
    main()


# -*- coding:utf-8 -*-
'''
__author__:liubin

'''
import unittest
from conf import settings
from lib import send_email

def suite():

    test_cases = unittest.defaultTestLoader.\
        discover(settings.TEST_PATH,pattern=settings.TEST_PATTERN,top_level_dir=None)

    # #装载测试用例
    # test_cases = unittest.TestLoader().loadTestsFromTestCase(TestStringMethods)
    #
    # #使用测试套件并打包测试用例
    # test_suit = unittest.TestSuite()
    # test_suit.addTests(test_cases)

    return test_cases

if __name__ == '__main__':

    test_result = unittest.TextTestRunner(verbosity=2).run(suite())

    print(test_result.testsRun)
    send_email.send_mail('hello.html')