import os
import bpy
from bpy_extras.io_utils import ImportHelper
from .drivers import update_drivers

def draw_panel(ctx, layout):
	layout.enabled = not ctx.ui_editing_mappings and not ctx.ui_editing_alignment and not ctx.setting_disable_drivers
	row = layout.row()
	row.prop(ctx, 'setting_bake_step', text='Frame Step')
	row.prop(ctx, 'setting_bake_linear', text='Linear Interpolation')
	layout.operator(BakingBakeOperator.bl_idname, icon='RENDER_ANIMATION')
	layout.operator(BakingBatchFBXImportOperator.bl_idname, icon='FILE_FOLDER')

def get_keyframes(obj):
	frames = []
	anim = obj.animation_data
	if anim is not None and anim.action is not None:
		for fcu in anim.action.fcurves:
			for keyframe in fcu.keyframe_points:
				x, y = keyframe.co
				if x not in frames:
					frames.append(x)

	return frames

def find_action(name):
	for action in bpy.data.actions:
		if action.name == name:
			return action

	return None

def transfer_anim(ctx):
	keyframes = get_keyframes(ctx.source)
	source_action = ctx.source.animation_data.action
	target_action_name = ctx.target.name + '|' + source_action.name.replace(ctx.source.name + '|', '')
	target_action = find_action(target_action_name)

	if target_action != None:
		while len(target_action.fcurves) > 0:
			target_action.fcurves.remove(target_action.fcurves[0])
	else:
		target_action = bpy.data.actions.new(target_action_name)

	ctx.target.animation_data.action = target_action

	bpy.ops.nla.bake(
		frame_start=int(min(keyframes)),
		frame_end=int(max(keyframes)),
		step=int(ctx.setting_bake_step),
		visual_keying=True,
		use_current_action=True,
		bake_types={'POSE'},
		only_selected=False
	)

	if ctx.setting_bake_linear:
		for fc in ctx.target.animation_data.action.fcurves:
			for kp in fc.keyframe_points:
				kp.interpolation = 'LINEAR'

	target_action.use_fake_user = True

class BakingBakeOperator(bpy.types.Operator):
	bl_idname = 'baking.bake'
	bl_label = 'Bake into Action'
	bl_description = 'Inserts animation keyframes transferred from the source based on the configured retargeting'

	def execute(self, context):
		ctx = context.object.retargeting_context
		transfer_anim(ctx)

		ctx.setting_disable_drivers = True
		update_drivers(ctx)

		context.window_manager.popup_menu(
			title='Bake Complete',
			icon='INFO',
			draw_func=lambda self, ctx: (
				self.layout.label(text='The retargeted animation has been successfully baked into the target armature. Drivers have been disabled so you can review the result animation')
			)
		)
		return {'FINISHED'}

class BakingBatchFBXImportOperator(bpy.types.Operator, ImportHelper):
	bl_idname = 'baking.batch_import'
	bl_label = 'Batch FBX Import & Bake'
	bl_description = 'Select multiple FBX files having the same source armature, and bake each file\'s animations into an Action on the target armature'
	directory: bpy.props.StringProperty(subtype='DIR_PATH')
	files: bpy.props.CollectionProperty(name='File paths', type=bpy.types.OperatorFileListElement)
	filter_glob: bpy.props.StringProperty(
		default='*.fbx',
		options={'HIDDEN'},
		maxlen=255
	)

	def execute(self, context):
		ctx = context.object.retargeting_context

		bpy.context.window_manager.progress_begin(0, len(self.files) * 2)
		progress = 0

		for file in self.files:
			bpy.ops.import_scene.fbx(
				filepath=os.path.join(self.directory, file.name),
				use_custom_props=True,
				use_custom_props_enum_as_string=True,
				ignore_leaf_bones=False,
				automatic_bone_orientation=True
			)

			bpy.context.window_manager.progress_update(progress)
			progress += 1

			imported_objects = []
			imported_source = None

			for obj in context.selected_objects:
				imported_objects.append(obj)

				if obj.type == 'ARMATURE':
					imported_source = obj

			for obj in imported_objects:
				obj.select_set(False)

			if imported_source != None:
				imported_action = imported_source.animation_data.action
				imported_source.scale = ctx.source.scale
				bpy.context.view_layer.objects.active = ctx.target
				ctx.target.select_set(True)
				prev = ctx.source
				ctx.selected_source = imported_source
				transfer_anim(ctx)
				ctx.selected_source = prev
				imported_source.animation_data.action = None
				bpy.data.actions.remove(imported_action)

			for obj in imported_objects:
				bpy.data.objects.remove(obj, do_unlink=True)

			bpy.context.window_manager.progress_update(progress)
			progress += 1

		bpy.context.window_manager.progress_end()

		return {'FINISHED'}

classes = (
	BakingBakeOperator,
	BakingBatchFBXImportOperator
)


from django.db.models import query
from django.shortcuts import render
from rest_framework.generics import GenericAPIView
from rest_framework.mixins import(ListModelMixin,CreateModelMixin,
                                  UpdateModelMixin,DestroyModelMixin,RetrieveModelMixin)
from api.models import Student
from api.serializers import StudentModelSerializer

from rest_framework.generics import (ListAPIView,CreateAPIView,
                                     UpdateAPIView,RetrieveAPIView,DestroyAPIView,
                                     RetrieveDestroyAPIView, RetrieveUpdateAPIView)

from rest_framework.throttling import ScopedRateThrottle

# Create your views here.
class StudentListMixin(GenericAPIView,ListModelMixin):
    queryset = Student.objects.all()
    # print("yesss")
    # breakpoint()
    serializer_class = StudentModelSerializer

    def get(self,request,*args,**kwargs):
        return self.list(request,*args,**kwargs)

class StudentCreateMixin(GenericAPIView,CreateModelMixin):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

    def post(self,request,*args,**kwargs):
        return self.create(request,*args,**kwargs)

class StudentRetriveMixin(GenericAPIView,RetrieveModelMixin):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

    def get(self,request,pk,*args,**kwargs):
        return self.retrieve(request,pk,*args,**kwargs)

class StudentUpdateMixin(GenericAPIView,UpdateModelMixin):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

    def post(self,request,pk,*args,**kwargs):
        return self.update(request,pk,*args,**kwargs)

class StudentDestroyMixin(GenericAPIView,DestroyModelMixin):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

    def delete(self,request,pk,*args,**kwargs):
        return self.destroy(request,pk,*args,**kwargs)

class StudentListAPIView(ListAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]
    throttle_scope = 'viewstu'

class StudentCreateAPIView(CreateAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]

class StudentUpdateAPIView(UpdateAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]
    throttle_scope ='modifystu'

class StudentRetriveAPIView(RetrieveAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]

class StudentDestoryAPIView(DestroyAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer

class StudentRetriveDestoryAPIView(RetrieveDestroyAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]

class StudentRetriveUpdateAPIView(RetrieveUpdateAPIView):
    queryset = Student.objects.all()
    serializer_class = StudentModelSerializer
    throttle_classes = [ScopedRateThrottle]


#############################################################################################################################
# Authors: Brandon Royal, Stephen Ulmer, Vien Yeung
#
# Description: Using the orginal image and the depth data image, determines the social relationship between individuals
#
# Change Log:
#   - Brandon Royal - Initial
#
#############################################################################################################################

from PIL import Image
import cv2
from CroppedFace import *
import numpy
import math

class DepthProcessing():

    def __init__(self, meeting, arrayStudents, size):
        self.meeting = meeting
        self.arrayStudents = arrayStudents
        self.size = size

    def beginDepthProcessing(self):

        cascPath = "haarcascade_frontalface_default.xml"
        faceCascade = cv2.CascadeClassifier(cascPath)

        ##open the images using Pil (use this if the shape idea does not work)
        #orginalImage = Image.open(self.meeting.getMeetingPicPath())
        #depthImage = Image.open(self.meeting.getDepthPicPath())
        ##get each images dimensions
        #orginalWidth, orginalHeight = orginalImage.size
        #depthHeight, depthWidth = depthImage.size

        #read images into open cv and detect the faces
        imageOrginal = cv2.imread(self.meeting.getMeetingPicPath())
        imageDepth = cv2.imread(self.meeting.getDepthPicPath(), 0)

        originalWidth, originalHeight = imageOrginal.shape[:2]
        depthWidth, depthHeight  = imageDepth.shape[:2]

        #print(originalWidth, originalHeight)
        #print(depthWidth, depthHeight)

        #determine the scaling multiplier based of the ratio of the depth to orginal width and height
        coordinateXMultiplier = depthWidth/originalWidth
        coordinateYMultiplier = depthHeight/originalHeight

        #gray = cv2.cvtColor(imageDepth, cv2.COLOR_BGR2GRAY)

        #img = cv2.imread(self.depthImagePath, 0) #since the image is grayscale, we need only one channel and the value '0' indicates just that
        #for i in range (img.shape[0]): #traverses through height of the image
        #    for j in range (img.shape[1]): #traverses through width of the image
        #        print(img[i][j])

        imageDepthWithDetection = imageDepth.copy()
        imageOrginalWithDetection = imageOrginal.copy()

        imageOrginal = cv2.cvtColor(imageOrginal, cv2.COLOR_BGR2GRAY)
        faces = faceCascade.detectMultiScale(imageOrginal, scaleFactor=1.2, minNeighbors=10, minSize=(30, 30))
        id = 0
        #print(imageDepth)
        #print(numpy.mean(imageDepth))
        croppedList = []
        for (x, y, w, h) in faces:

            #first lets crop the faces from the meeting pic and save them for later use
            cropImg = imageOrginal[y:y+h, x:x+w]
            #gray = cv2.cvtColor(cropImg, cv2.COLOR_BGR2GRAY)  # turn portrait to grayscale
            shrink = cv2.resize(cropImg, (self.size, self.size))
            cropImgPath = self.meeting.getCropsDirectory()+"//"+str(id)+'.jpg'
            cv2.imwrite(cropImgPath, shrink)
            #print("wrote cropped face " + str(id))

            #next we get the face coordinates for the depth image based off the multiplier found earlier
            xs = int(x*coordinateXMultiplier)
            xws = int((x+w)*coordinateXMultiplier)
            ys = int(y*coordinateYMultiplier)
            yhs = int((y+h)*coordinateYMultiplier)
            #print("Depth Face Bounds")
            #print(xs, xws, ys, yhs)

            #save a copy of the meeting pic with detection info
            imageOrginalWithDetection = cv2.rectangle(imageOrginalWithDetection, (x, y), (x+w, y+h), (0, 255, 0), 5)
            imageOrginalWithDetection = cv2.putText(imageOrginalWithDetection, 'F' + str(id), (x,y), cv2.FONT_HERSHEY_COMPLEX, 1.5, (0, 255, 255), 5)

            count = 0
            total = 0
            #print("i:")
            #print(len(imageDepth))
            #print(xs, xws)
            #print()
            #print("j:")
            #print(len(imageDepth[0]))
            #print(ys, yhs)
            for i in range (xs, xws): #traverses through height of the image
                for j in range (ys, yhs): #traverses through width of the image
                    count = count + 1
                    total = total + imageDepth[j][i]
                    #print(count, gray[i][j])
            averageGreyScale = total/count
            distanceCameraToStudent = 1/(averageGreyScale/255)

            distanceFeet = 3.28084 * distanceCameraToStudent * 2
            distanceInches = distanceFeet * 12
            #print(total, count, averageGreyScale, distanceCameraToStudent, distanceFeet, distanceInches)

            #save a copy of the depth pic with detection info

            imageDepthWithDetection = cv2.rectangle(imageDepthWithDetection, (int(x*coordinateXMultiplier), int(y*coordinateYMultiplier)), (int((x+w)*coordinateXMultiplier), int((y+h)*coordinateYMultiplier)), (0, 255, 0), 2)
            imageDepthWithDetection = cv2.putText(imageDepthWithDetection, 'F' + str(id), (int(x*coordinateXMultiplier),int(y*coordinateYMultiplier)), cv2.FONT_HERSHEY_COMPLEX, .3, (0, 255, 255), 1)

            faceMidPointDepth = int((xs+(xws/2))), int((ys+(yhs/2)))

            faceMidPointOriginal = int(x+(w/2)), int(y+(h/2))

            xm = int((xs+(xws/2)))
            ym = int((ys+(yhs/2)))

            #print("Midpoint: " + str(faceMidPoint))

            #path = str(id) + ".jpg"
            croppedFace = CroppedFace(id, x, y, w, h, int(x*coordinateXMultiplier), int(y*coordinateYMultiplier), int((x+w)*coordinateXMultiplier), int((y+h)*coordinateYMultiplier), cropImgPath, averageGreyScale, faceMidPointOriginal, distanceCameraToStudent)
            croppedList.append(croppedFace)
            id = id + 1

        #cv2.imwrite("orginalDetection.jpg", imageOrginal)
        #cv2.imwrite("depthDetection.jpg", imageDepth)
        cv2.imwrite(self.meeting.getMeetingDirectory() + "//" + "meetingPicWithDetection.jpg", imageOrginalWithDetection)
        cv2.imwrite(self.meeting.getMeetingDirectory() + "//" + "depthPicWithDetection.jpg", imageDepthWithDetection)
        #print("wrote meeting pic and depth pic with detection")

        self.meeting.setCroppedFaces(croppedList)
        finalDistanceMatrix = self.createDistanceMatrix(croppedList)
        normalizedDistanceMatrix = self.normalizeSocialData(finalDistanceMatrix)
        self.meeting.setUnrecognizedSocialMatrix(normalizedDistanceMatrix)

    def createDistanceMatrix(self, croppedList):

        degreesPerPixel = .0108

        numStudents = len(croppedList)
        distanceMatrix = [[0 for x in range(numStudents)] for y in range(numStudents)]
        for face in croppedList:
            for otherFace in croppedList:
                if(face.getId() == otherFace.getId()):
                    continue
                pixelsBetween = self.pixelsBetweenFaces(face.getMidPoint()[0], face.getMidPoint()[1], otherFace.getMidPoint()[0], otherFace.getMidPoint()[1])
                theta = pixelsBetween * degreesPerPixel
                finalDistanceBetween = self.getActualDistanceBetweenFaces(face.getDistanceFromCamera(), otherFace.getDistanceFromCamera(), theta)
                distanceMatrix[face.getId()][otherFace.getId()] = finalDistanceBetween
                #print(pixelsBetween, theta)
                #print("Distance Between " + str(face.getId()) + " and " + str(otherFace.getId()) + " is " + str(finalDistanceBetween))
                #print()

        return distanceMatrix

    def pixelsBetweenFaces(self, x1, y1, x2, y2):
        a = abs(x2-x1)
        b = abs(y2-y1)
        c = math.sqrt((a*a) + (b*b))
        return c

    def getActualDistanceBetweenFaces(self, a, b, theta):
        radians = theta * 0.0174533
        return math.sqrt((a*a)+(b*b)-(2*a*b*math.cos(radians)))

    def normalizeSocialData(self, distanceMatrix):
        numStudents = len(distanceMatrix)
        allDistances = []

        #put all distances inside a one dimensional array
        for i in range(numStudents):
            for j in range(numStudents):
                if(distanceMatrix[i][j] != 0):
                    allDistances.append(distanceMatrix[i][j])

        #sort the array
        allDistances.sort()

        #find the three partitions of the array
        first = 0
        second = int(((numStudents*numStudents) - numStudents)/3)
        third = (second*2)

        firstMin = 0
        firstMax = allDistances[second-1]
        secondMin = allDistances[second]
        secondMax = allDistances[third-1]
        thirdMin = allDistances[third]

        #print(allDistances)

        normalizedMatrix = [[0 for x in range(numStudents)] for y in range(numStudents)]

        for i in range(numStudents):
            for j in range(numStudents):
                value = distanceMatrix[i][j]
                if(value <= firstMax and value > 0):
                    normalizedMatrix[i][j] = 1
                elif(value >= secondMin and value <= secondMax):
                    normalizedMatrix[i][j] = 2
                elif(value >= thirdMin):
                    normalizedMatrix[i][j] = 3
        #print()

        #for i in range(numStudents):
        #    print(distanceMatrix[i])

        #print()
        #for i in range(numStudents):
        #    print(normalizedMatrix[i])

        return normalizedMatrix


# -*- coding=utf-8 -*-
"""
#   Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
from __future__ import print_function
import os
import time
import numpy as np
import logging
import argparse

import paddle
import paddle.fluid as fluid
from paddle.fluid import profiler

from args import print_arguments, parse_args
from infer_network import TdmInferNet
from dataset_generator import TDMDataset

logging.basicConfig(format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger("fluid")
logger.setLevel(logging.INFO)

def to_tensor(data, place):
    """
    Convert data to paddle tensor
    """
    flattened_data = np.concatenate(data, axis=0).astype("float32")
    flattened_data = flattened_data.reshape([-1, 768])
    res = fluid.Tensor()
    res.set(flattened_data, place)
    return res

def data2tensor(data, place):
    """
    Dataset prepare
    """
    input_emb = to_tensor([x[0] for x in data], place)

    return input_emb

def run_infer(args, model_path):
    """run infer"""
    logger.info("Infer Begin")
    file_list = [
        str(args.test_files_path) + "/%s" % x
        for x in os.listdir(args.test_files_path)
    ]

    tdm_model = TdmInferNet(args)
    inputs = tdm_model.input_data()
    res_item = tdm_model.infer_net(inputs)
    test_reader = TDMDataset().infer_reader(file_list, args.batch_size)

    place = fluid.CPUPlace()
    exe = fluid.Executor(place)

    path = os.path.join(args.model_files_path, model_path)
    fluid.io.load_persistables(
        executor=exe,
        dirname=path,
        main_program=fluid.default_main_program())

    logger.info("Load persistables from \"{}\"".format(path))

    if args.save_init_model:
        logger.info("Begin Save infer model.")
        model_path = (str(args.model_files_path) + "/" + "infer_model")
        fluid.io.save_inference_model(executor=exe, dirname=model_path,
                                      feeded_var_names=[
                                          'input_emb', 'first_layer_node', 'first_layer_node_mask'],
                                      target_vars=[res_item])
        logger.info("End Save infer model.")

    first_layer_node = tdm_model.first_layer_node
    first_layer_nums = len(first_layer_node)
    first_layer_node = np.array(first_layer_node)
    first_layer_node = first_layer_node.reshape((1, -1)).astype('int64')
    first_layer_node = first_layer_node.repeat(args.batch_size, axis=0)
    # 在demo中，假设infer起始层的节点都不是叶子节点，mask=0
    # 若真实的起始层含有叶子节点，则对应位置的 mask=1
    first_layer_mask = (
        np.zeros((args.batch_size, first_layer_nums))).astype('int64')

    for batch_id, data in enumerate(test_reader()):
        input_emb = data2tensor(data, place)
        item_res = exe.run(fluid.default_main_program(),
                           feed={"input_emb": input_emb,
                                 "first_layer_node": first_layer_node,
                                 "first_layer_node_mask": first_layer_mask},
                           fetch_list=[res_item])
        logger.info("TEST --> batch: {} infer_item {}".format(
            batch_id, item_res))
    logger.info("Inference complete!")

if __name__ == "__main__":
    args = parse_args()
    print_arguments(args)
    # 在此处指定infer模型所在的文件夹
    path = "epoch_0"
    run_infer(args, path)


import numpy as np
import matplotlib
import matplotlib.pyplot as plt
xf = np.loadtxt('logistic_x.txt')
y = np.loadtxt('logistic_y.txt')
colors = ['red', 'blue']
plt.scatter(xf[:, 0], xf[:, 1], c=y,cmap=matplotlib.colors.ListedColormap(colors))
plt.xlabel('x1')
plt.ylabel('x2')
m=y.shape[0]
n=xf.shape[1]
theta=np.zeros((n+1,1))
print (theta)
x=np.insert(xf,0,1,axis=1)
print (x)
print (y)
def g(t):
    return 1/(1+np.exp(-t))
def h(a):
    return np.dot(a,theta)
def costj(theta):
    l=0
    for i in range(m):
        u = y[i]*h(x[i, :])
        l-=np.log(u)/m
    return l
def gradient(l):
    grad=0
    for i in range(m):
        grad -= (1-g(y[i]*h(x[i, :])))*y[i]*x[i, :]/m
        return grad
def hessian(l):
    hess=np.zeros((n+1,n+1))
    for i in range(m):
        xi=np.dot(x[i,:],np.transpose(x[i,:]))
        hess -= g(y[i]*h(x[i, :])) * (1-g(y[i]*h(x[i, :])))*xi/m
        return hess
maxiters=10
for k in range(maxiters):
    theta=theta-np.matmul(np.linalg.pinv(hessian(l=costj(theta))),gradient(l=costj(theta)))
    print (theta)
p=np.zeros(m,1)
plot_y=np.zeros(m,1)
for i in range(m):
    p[i]= g(y[i]*h(x[i, :]))
    if p>0.5:
        plot_y[i]=1
    else:
        plot_y[i]=-1

plt.plot(xf,plot_y)
plt.show()


import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
from os import listdir
from os.path import isfile, join, isdir
import argparse
#Get files from exp directory
templates = [
            '(ROOT(S(NP)(VP)(.)))' ,

            '(ROOT(S(VP)(.)))' ,

            '(ROOT(NP(NP)(.)))' ,

            '(ROOT(FRAG(SBAR)(.)))' ,

            '(ROOT(S(S)(,)(CC)(S)(.)))' ,

            '(ROOT(SBARQ(WHADVP)(SQ)(.)))' ,

            '(ROOT(S(PP)(,)(NP)(VP)(.)))' ,

            '(ROOT(S(ADVP)(NP)(VP)(.)))' ,

            '(ROOT(S(SBAR)(,)(NP)(VP)(.)))'
        ]
def get_values(df):
    df1=df.query("Inc==1")
    #df2=df.query("Inc==0")
    index=df1.index

    print(index)
    X=[]
    Y=[]
    #s= df1["ID"].sum()
    for i in range(len(index)):
        x1=index[i][0]
        x2=index[i][1]
        df2=df.query("Dif=="+str(x1))
        s = df2["ID"].sum ()
        print(s)
        y=(df1.loc[x1,x2]["ID"])

        X.append(x1)
        Y.append(y/s)
    return X,Y

def inc_pars(args):
    mypath = args.log_path
    out_path=args.out_path
    dirs = [dI for dI in listdir (mypath) if isdir (join (mypath , dI))]
    datasets = ["AMAZON_PRODUCTS" , "NEWS_HEADLINES" , "STANFORD_TREEBANK"]
    tools = ["CHAR_CNN" , "CNN_TEXT" , "SENTICNET" , "SENTIWORDNET" , "STANFORD" , "VADER"]
    fig = plt.figure ()
    # sns.set (style="whitegrid")
    sns.set (font_scale=0.6)

    # sns.palplot (sns.hls_palette (8 , l=.3 , s=.8))
    numfile = 0
    clusters_tab=[]
    for dir in dirs:
        path_dir = join (mypath , dir)
        onlyfiles = [f for f in listdir (path_dir) if isfile (join (path_dir , f))]
        numfile = numfile + 1
        tool_cluster=pd.DataFrame();
        i=0;
        for f in onlyfiles:
            print(f)

            path_file=join(path_dir,f)
            df= pd.read_csv(path_file,sep=";")
            #df=df.loc[df["Dif"]<20]
            #df["Dif"]= df["Dif"].apply(lambda x: round(x,2))
            #clusters=df.groupby(["Parse"]).agg("count")
            clusters = df.groupby (["Parse"], as_index=False)["Inc"].mean()
            clusters.columns=["Parse","Inc"]
            tool_cluster = clusters
            """
            if i==0:
                tool_cluster= clusters
            else:
                tool_cluster=pd.merge (tool_cluster , clusters,how="inner",on="Parse")

                #tool_cluster=pd.merge(tool_cluster,clusters,how="inner", on=["Parse"])
                #tool_cluster=tool_cluster.dropna (inplace=True)

            i=i+1
        tool_cluster.index.names=["Parse"]

        """

        df_new= pd.DataFrame(columns=tool_cluster.columns)
        j=0;
        for i in range(len(tool_cluster)):

            if tool_cluster.iloc[i]["Parse"] in templates:
                df_new.loc[j]=tool_cluster.iloc[i]
                j=j+1

        df_new.to_csv(join(out_path,f),sep=";")

        """
        X,Y=get_values (clusters)
        #sns.scatterplot (X , Y ,ax=ax)            #clusters= list(clusters)
        #y1=df["Inc"].apply(lambda x: round(x,2))
        ax = fig.add_subplot (1, 6 , numfile)
        #sns.lineplot (X,Y , ax=ax)
        sns.scatterplot (X , Y , ax=ax)
        j=j+1
        #print(X)
        #print(y1)
        """
        #print(clusters.loc[clusters["Id"]>10])

        """
        for cluster in clusters:

          print(cluster[1])
        """
        """
        ax.set_ylabel ("nb_inconsistencies" , fontsize=10)
        ax.set_xlabel ("diff in words" , fontsize=10)
        ax.set_title (dir , fontsize=10)
        """

    #plt.figlegend ( datasets , loc='lower center' , ncol=3 , labelspacing=0.)
    #plt.show()
    #fig.savefig (args.out_path)

if __name__ == '__main__':
    ##model args
    parser = argparse.ArgumentParser (description='Syntactically Controlled Paraphrase Transformer')

    parser.add_argument ('--log_path' , type=str , default="D:/Users/wissam/Documents/These/these/papers_material/VLDB_submission/experiments/logs/Intool_inc_sim/sim_inc" ,
                         help='input path of logs')

    parser.add_argument ('--out_path' , type=str , default='./data/sentiment_dataset_sentic.csv' ,
                         help='output of plots')

    args = parser.parse_args ()
    inc_pars(args)


from flask import Flask, jsonify, render_template, request
from flask_sslify import SSLify
import json
import os

app = Flask(__name__)
sslify = SSLify(app)

VERSION = '1.1';
VERSION_KEY = 'version_key';

CURRENT_DIRECTORY = os.path.dirname(__file__)
PATH_JSON_COURSES = os.path.join(CURRENT_DIRECTORY, 'data', 'courses.json')

with open(PATH_JSON_COURSES, 'r') as f:
    course_dictionary = json.load(f)

@app.route('/course/<string:course_code>', methods=['GET'])
def get_task(course_code):
    if course_code in course_dictionary:
        return jsonify({
            'code': course_code,
            'name': course_dictionary[course_code]
        })
    else:
        return jsonify({
            'code':course_code,
            'name': ''
        })

@app.route('/courses', methods=['GET'])
def get_courses():
    all_args = request.args.to_dict()

    response_dict = {}

    if VERSION_KEY in all_args:
        response_dict[VERSION_KEY] = VERSION
        # Delete the query key so we don't iterate over it below
        del all_args[VERSION_KEY]

    for _, course_code in all_args.items():
        if course_code in course_dictionary:
            response_dict[course_code] = course_dictionary[course_code]

    return jsonify(response_dict)

@app.route('/')
def index():
    return render_template('index.html')

# CORS
@app.after_request
def apply_caching(response):
    response.headers['Access-Control-Allow-Origin'] = 'https://ssc.adm.ubc.ca'
    response.headers['Access-Control-Allow-Methods'] = 'GET'
    response.headers['Access-Control-Allow-Headers'] = 'Content-Type'
    return response

if __name__ == '__main__':
    app.run()


from fairseq.models import FairseqEncoder

import torch
import torch.nn as nn
import math

from fairseq import options, utils

from fairseq.models.fairseq_encoder import EncoderOut
from fairseq.modules import (
    AdaptiveSoftmax,
    FairseqDropout,
    LayerDropModuleList,
    LayerNorm,
    PositionalEmbedding,
    SinusoidalPositionalEmbedding,
    TransformerEncoderLayer,
)

from fairseq.modules.quant_noise import quant_noise as apply_quant_noise_
from torch import Tensor

class SrcFactorEncoder(FairseqEncoder):
    """
    Based on TransformerEncoder but adds an additional
    embedding for src_factor.
    Changes have comments
    """
    def __init__(self, args, dictionary, embed_tokens, src_factor_tokens):
        super().__init__(dictionary)

        self.register_buffer("version", torch.Tensor([3]))

        self.dropout_module = FairseqDropout(args.dropout, module_name=self.__class__.__name__)
        self.encoder_layerdrop = args.encoder_layerdrop
        """
        change the embedding dim to account for src_factor
        """
        embed_dim = embed_tokens.embedding_dim + src_factor_tokens.embedding_dim

        self.padding_idx = embed_tokens.padding_idx
        self.max_source_positions = args.max_source_positions

        self.embed_tokens = embed_tokens
        """
        save embedding
        """
        self.src_factor_tokens = src_factor_tokens

        self.embed_scale = 1.0 if args.no_scale_embedding else math.sqrt(embed_dim)

        self.embed_positions = (
            PositionalEmbedding(
                args.max_source_positions,
                embed_dim,
                self.padding_idx,
                learned=args.encoder_learned_pos,
            )
            if not args.no_token_positional_embeddings
            else None
        )

        if not args.adaptive_input and args.quant_noise_pq > 0:
            self.quant_noise = apply_quant_noise_(
                nn.Linear(embed_dim, embed_dim, bias=False),
                args.quant_noise_pq,
                args.quant_noise_pq_block_size,
            )
        else:
            self.quant_noise = None

        if self.encoder_layerdrop > 0.0:
            self.layers = LayerDropModuleList(p=self.encoder_layerdrop)
        else:
            self.layers = nn.ModuleList([])
        self.layers.extend(
            [self.build_encoder_layer(args) for i in range(args.encoder_layers)]
        )
        self.num_layers = len(self.layers)

        if args.encoder_normalize_before:
            self.layer_norm = LayerNorm(embed_dim)
        else:
            self.layer_norm = None
        if getattr(args, "layernorm_embedding", False):
            self.layernorm_embedding = LayerNorm(embed_dim)
        else:
            self.layernorm_embedding = None

    def forward_embedding(self, src_tokens, src_factor_tokens):
        """
        concat before passing to the rest
        """
        x = self.embed_tokens(src_tokens)
        x = torch.cat([self.src_factor_tokens(src_factor_tokens),x],-1)

        x = embed = self.embed_scale * x
        if self.embed_positions is not None:
            x = embed + self.embed_positions(src_tokens)
        if self.layernorm_embedding is not None:
            x = self.layernorm_embedding(x)
        x = self.dropout_module(x)
        if self.quant_noise is not None:
            x = self.quant_noise(x)
        return x, embed

    def build_encoder_layer(self, args):
        return TransformerEncoderLayer(args)

    def forward(self, src_tokens, src_lengths, src_factor_tokens, src_factor_lengths, return_all_hiddens: bool = False):
        """
        forward embedding signature
        """
        x, encoder_embedding = self.forward_embedding(src_tokens, src_factor_tokens)

        # B x T x C -> T x B x C
        x = x.transpose(0, 1)
        # compute padding mask
        encoder_padding_mask = src_tokens.eq(self.padding_idx)

        encoder_states = [] if return_all_hiddens else None

        # encoder layers
        for layer in self.layers:
            x = layer(x, encoder_padding_mask)
            if return_all_hiddens:
                assert encoder_states is not None
                encoder_states.append(x)

        if self.layer_norm is not None:
            x = self.layer_norm(x)

        return EncoderOut(
            encoder_out=x,  # T x B x C
            encoder_padding_mask=encoder_padding_mask,  # B x T
            encoder_embedding=encoder_embedding,  # B x T x C
            encoder_states=encoder_states,  # List[T x B x C]
            src_tokens=None,
            src_lengths=None,
        )

    @torch.jit.export
    def reorder_encoder_out(self, encoder_out: EncoderOut, new_order):
        """
        Reorder encoder output according to *new_order*.

        Args:
            encoder_out: output from the ``forward()`` method
            new_order (LongTensor): desired order

        Returns:
            *encoder_out* rearranged according to *new_order*
        """
        """
        Since encoder_padding_mask and encoder_embedding are both of type
        Optional[Tensor] in EncoderOut, they need to be copied as local
        variables for Torchscript Optional refinement
        """
        encoder_padding_mask: Optional[Tensor] = encoder_out.encoder_padding_mask
        encoder_embedding: Optional[Tensor] = encoder_out.encoder_embedding

        new_encoder_out = (
            encoder_out.encoder_out
            if encoder_out.encoder_out is None
            else encoder_out.encoder_out.index_select(1, new_order)
        )
        new_encoder_padding_mask = (
            encoder_padding_mask
            if encoder_padding_mask is None
            else encoder_padding_mask.index_select(0, new_order)
        )
        new_encoder_embedding = (
            encoder_embedding
            if encoder_embedding is None
            else encoder_embedding.index_select(0, new_order)
        )
        src_tokens = encoder_out.src_tokens
        if src_tokens is not None:
            src_tokens = src_tokens.index_select(0, new_order)

        src_lengths = encoder_out.src_lengths
        if src_lengths is not None:
            src_lengths = src_lengths.index_select(0, new_order)

        encoder_states = encoder_out.encoder_states
        if encoder_states is not None:
            for idx, state in enumerate(encoder_states):
                encoder_states[idx] = state.index_select(1, new_order)

        return EncoderOut(
            encoder_out=new_encoder_out,  # T x B x C
            encoder_padding_mask=new_encoder_padding_mask,  # B x T
            encoder_embedding=new_encoder_embedding,  # B x T x C
            encoder_states=encoder_states,  # List[T x B x C]
            src_tokens=src_tokens,  # B x T
            src_lengths=src_lengths,  # B x 1
        )

    def max_positions(self):
        """Maximum input length supported by the encoder."""
        if self.embed_positions is None:
            return self.max_source_positions
        return min(self.max_source_positions, self.embed_positions.max_positions)

    def upgrade_state_dict_named(self, state_dict, name):
        """Upgrade a (possibly old) state dict for new versions of fairseq."""
        if isinstance(self.embed_positions, SinusoidalPositionalEmbedding):
            weights_key = "{}.embed_positions.weights".format(name)
            if weights_key in state_dict:
                print("deleting {0}".format(weights_key))
                del state_dict[weights_key]
            state_dict[
                "{}.embed_positions._float_tensor".format(name)
            ] = torch.FloatTensor(1)
        for i in range(self.num_layers):
            # update layer norms
            self.layers[i].upgrade_state_dict_named(
                state_dict, "{}.layers.{}".format(name, i)
            )

        version_key = "{}.version".format(name)
        if utils.item(state_dict.get(version_key, torch.Tensor([1]))[0]) < 2:
            # earlier checkpoints did not normalize after the stack of layers
            self.layer_norm = None
            self.normalize = False
            state_dict[version_key] = torch.Tensor([1])
        return state_dict


#!/usr/bin/env python

import argparse
import datetime
import os
import shutil
import cv2
from signal import signal, SIGINT

import imutils
from imutils.video import FPS

from config import WildlifeConfig
from motion import MotionDetection
from notifier import TelegramNotifier
from video_writer import AsyncVideoWriter

exit_by_handler = False

def signal_handler(signal_received, f):
    global exit_by_handler
    print('[INFO] SIGINT or CTRL-C detected. Exiting gracefully')
    exit_by_handler = True

def create_video_filename(start_time, path):
    return "{}/{}-wildlife.avi".format(path, start_time.strftime("%Y%m%d-%H%M%S"))

start_recording_threshold_t1 = None
start_recording_threshold_t2 = None

def start_recording_threshold(activity_count):
    global start_recording_threshold_t1
    global start_recording_threshold_t2
    if activity_count % 5 == 0:
        start_recording_threshold_t2 = start_recording_threshold_t1
        start_recording_threshold_t1 = datetime.datetime.now()
        if start_recording_threshold_t2 is not None and (
                start_recording_threshold_t1 - start_recording_threshold_t2).seconds < 1:
            return True
    return False

last_activity = None

recording_status = "OFF"
recording_color = (255, 255, 225)
recording_info = ""
recording_filename = ""
start_recording_time = None
stop_recording_time = None
video_out = None
activity_count_total = 0
activity_count_during_recording = 0
last_recording_snapshot = None

signal(SIGINT, signal_handler)

ap = argparse.ArgumentParser()
ap.add_argument("-c", "--config", required=True,
                help="path to the JSON configuration file")
args = vars(ap.parse_args())

config = WildlifeConfig(args["config"])

# prepare video storage folder
if config.clean_store_on_startup:
    shutil.rmtree(config.store_path)
os.mkdir(config.store_path)

if config.system == "raspberrypi":
    from capture_picamera import CapturePiCameraAsync as Capture
else:
    from capture_opencv import CaptureOpencv as Capture

notifier = None
if config.telegram_notification:
    notifier = TelegramNotifier(config)

def writer_finished(file_name):
    global notifier
    if notifier is not None and last_recording_snapshot is not None:
        snapshot_filename = file_name.rsplit('.', 1)[0] + '.jpg'
        cv2.imwrite(snapshot_filename, last_recording_snapshot)
        notifier.send_message("New Wildlife Video: {}".format(os.path.basename(file_name)), snapshot_filename)

capture = Capture(config)

writer = AsyncVideoWriter(config, writer_finished)

motion = MotionDetection(config)

motion_detected = False
motion_rectangles = [(0, 0, config.resolution[0], config.resolution[1])]

capture.start()

fps = FPS().start()

frame_count = 0
while True:
    frame, frame_timestamp = capture.read()

    if frame is None:
        continue

    frame_count += 1

    timestamp = datetime.datetime.now()

    if config.motion_detection and frame_count % 3 == 0:
        motion_detected, motion_rectangles = motion.detect_motion(frame)

    motion_status = "activity"
    motion_status_color = (255, 255, 255)
    if motion_detected:
        last_activity = datetime.datetime.now()
        activity_count_total += 1

        if recording_status == "OFF" and start_recording_threshold(activity_count_total):
            recording_status = "ON"
            recording_color = (0, 0, 255)
            start_recording_time = frame_timestamp
            recording_filename = create_video_filename(start_recording_time, config.store_path)
            if config.store_video:
                activity_count_during_recording = 0
                writer.start(recording_filename)

        activity_count_during_recording += 1
        if activity_count_during_recording == config.store_activity_count_threshold + 1:
            last_recording_snapshot = frame.copy()

        motion_status = "activity"
        motion_status_color = (0, 255, 0)
        if motion_rectangles is not None:
            for r in motion_rectangles:
                cv2.rectangle(frame, (r[0], r[1]), (r[0] + r[2], r[1] + r[3]), motion_status_color, 1)

    if recording_status == "ON" and last_activity < timestamp and \
            (timestamp - last_activity).seconds >= config.min_recording_time_seconds:
        if config.store_video:
            writer.stop(activity_count_during_recording)
        recording_status = "OFF"
        recording_info = ""
        stop_recording_time = timestamp
        recording_color = (255, 255, 255)

    if config.store_video and recording_status == "ON":
        recording_info = " | " + recording_filename + " " + str((frame_timestamp - start_recording_time).seconds) + \
                         " activity: " + str(activity_count_during_recording)

    timestamp_str = frame_timestamp.strftime("%A %d %B %Y %I:%M:%S%p")
    cv2.putText(frame, motion_status, (10, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.5, motion_status_color, 2)
    cv2.putText(frame, recording_status, (frame.shape[1] - 50, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.35, recording_color, 2)
    cv2.putText(frame, timestamp_str + recording_info, (10, frame.shape[0] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.35
                , (255, 255, 255), 1)

    # store frame
    if config.store_video:
        writer.write(frame)

    if config.show_video:
        cv2.imshow('captured frame', frame)
        if cv2.waitKey(1) == ord('q'):
            break

    if exit_by_handler:
        break

    fps.update()

# shutdown
fps.stop()
print("[INFO] elapsed time: {:.2f} s".format(fps.elapsed()))
print("[INFO] approx. FPS: {:.2f}".format(fps.fps()))
writer.stop(0)
capture.stop()
if config.show_video:
    cv2.destroyAllWindows()


#!/usr/bin/python
# Copyright (c) 2017 Ansible Project
# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)

from __future__ import (absolute_import, division, print_function)
__metaclass__ = type

ANSIBLE_METADATA = {'metadata_version': '1.1',
                    'status': ['preview'],
                    'supported_by': 'community'}

DOCUMENTATION = '''
---
module: luks_device

short_description: Manage encrypted (LUKS) devices

version_added: "2.8"

description:
    - "Module manages L(LUKS,https://en.wikipedia.org/wiki/Linux_Unified_Key_Setup)
      on given device. Supports creating, destroying, opening and closing of
      LUKS container and adding or removing new keys."

options:
    device:
        description:
            - "Device to work with (e.g. C(/dev/sda1)). Needed in most cases.
              Can be omitted only when I(state=closed) together with I(name)
              is provided."
        type: str
    state:
        description:
            - "Desired state of the LUKS container. Based on its value creates,
              destroys, opens or closes the LUKS container on a given device."
            - "I(present) will create LUKS container unless already present.
              Requires I(device) and I(keyfile) options to be provided."
            - "I(absent) will remove existing LUKS container if it exists.
              Requires I(device) or I(name) to be specified."
            - "I(opened) will unlock the LUKS container. If it does not exist
              it will be created first.
              Requires I(device) and I(keyfile) to be specified. Use
              the I(name) option to set the name of the opened container.
              Otherwise the name will be generated automatically and returned
              as a part of the result."
            - "I(closed) will lock the LUKS container. However if the container
              does not exist it will be created.
              Requires I(device) and I(keyfile) options to be provided. If
              container does already exist I(device) or I(name) will suffice."
        type: str
        default: present
        choices: [present, absent, opened, closed]
    name:
        description:
            - "Sets container name when I(state=opened). Can be used
              instead of I(device) when closing the existing container
              (i.e. when I(state=closed))."
        type: str
    keyfile:
        description:
            - "Used to unlock the container and needed for most
              of the operations. Parameter value is the path
              to the keyfile with the passphrase."
            - "BEWARE that working with keyfiles in plaintext is dangerous.
              Make sure that they are protected."
        type: path
    new_keyfile:
        description:
            - "Adds additional key to given container on I(device).
              Needs I(keyfile) option for authorization. LUKS container
              supports up to 8 keys. Parameter value is the path
              to the keyfile with the passphrase."
            - "NOTE that adding additional keys is I(not idempotent).
              A new keyslot will be used even if another keyslot already
              exists for this keyfile."
            - "BEWARE that working with keyfiles in plaintext is dangerous.
              Make sure that they are protected."
        type: path
    remove_keyfile:
        description:
            - "Removes given key from the container on I(device). Does not
              remove the keyfile from filesystem.
              Parameter value is the path to the keyfile with the passphrase."
            - "NOTE that removing keys is I(not idempotent). Trying to remove
              a key which no longer exists results in an error."
            - "NOTE that to remove the last key from a LUKS container, the
              I(force_remove_last_key) option must be set to C(yes)."
            - "BEWARE that working with keyfiles in plaintext is dangerous.
              Make sure that they are protected."
        type: path
    force_remove_last_key:
        description:
            - "If set to C(yes), allows removing the last key from a container."
            - "BEWARE that when the last key has been removed from a container,
              the container can no longer be opened!"
        type: bool
        default: no

requirements:
    - "cryptsetup"
    - "wipefs"
    - "lsblk"

author:
    "Jan Pokorny (@japokorn)"
'''

EXAMPLES = '''

- name: create LUKS container (remains unchanged if it already exists)
  luks_device:
    device: "/dev/loop0"
    state: "present"
    keyfile: "/vault/keyfile"

- name: (create and) open the LUKS container; name it "mycrypt"
  luks_device:
    device: "/dev/loop0"
    state: "opened"
    name: "mycrypt"
    keyfile: "/vault/keyfile"

- name: close the existing LUKS container "mycrypt"
  luks_device:
    state: "closed"
    name: "mycrypt"

- name: make sure LUKS container exists and is closed
  luks_device:
    device: "/dev/loop0"
    state: "closed"
    keyfile: "/vault/keyfile"

- name: create container if it does not exist and add new key to it
  luks_device:
    device: "/dev/loop0"
    state: "present"
    keyfile: "/vault/keyfile"
    new_keyfile: "/vault/keyfile2"

- name: add new key to the LUKS container (container has to exist)
  luks_device:
    device: "/dev/loop0"
    keyfile: "/vault/keyfile"
    new_keyfile: "/vault/keyfile2"

- name: remove existing key from the LUKS container
  luks_device:
    device: "/dev/loop0"
    remove_keyfile: "/vault/keyfile2"

- name: completely remove the LUKS container and its contents
  luks_device:
    device: "/dev/loop0"
    state: "absent"
'''

RETURN = '''
name:
    description:
        When I(state=opened) returns (generated or given) name
        of LUKS container. Returns None if no name is supplied.
    returned: success
    type: str
    sample: "luks-c1da9a58-2fde-4256-9d9f-6ab008b4dd1b"
'''

import os
import re
import stat

from ansible.module_utils.basic import AnsibleModule

RETURN_CODE = 0
STDOUT = 1
STDERR = 2

# used to get <luks-name> out of lsblk output in format 'crypt <luks-name>'
# regex takes care of any possible blank characters
LUKS_NAME_REGEX = re.compile(r'\s*crypt\s+([^\s]*)\s*')
# used to get </luks/device> out of lsblk output
# in format 'device: </luks/device>'
LUKS_DEVICE_REGEX = re.compile(r'\s*device:\s+([^\s]*)\s*')

class Handler(object):

    def __init__(self, module):
        self._module = module
        self._lsblk_bin = self._module.get_bin_path('lsblk', True)

    def _run_command(self, command):
        return self._module.run_command(command)

    def generate_luks_name(self, device):
        ''' Generate name for luks based on device UUID ('luks-<UUID>').
            Raises ValueError when obtaining of UUID fails.
        '''
        result = self._run_command([self._lsblk_bin, '-n', device, '-o', 'UUID'])

        if result[RETURN_CODE] != 0:
            raise ValueError('Error while generating LUKS name for %s: %s'
                             % (device, result[STDERR]))
        dev_uuid = result[STDOUT].strip()
        return 'luks-%s' % dev_uuid

class CryptHandler(Handler):

    def __init__(self, module):
        super(CryptHandler, self).__init__(module)
        self._cryptsetup_bin = self._module.get_bin_path('cryptsetup', True)

    def get_container_name_by_device(self, device):
        ''' obtain LUKS container name based on the device where it is located
            return None if not found
            raise ValueError if lsblk command fails
        '''
        result = self._run_command([self._lsblk_bin, device, '-nlo', 'type,name'])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while obtaining LUKS name for %s: %s'
                             % (device, result[STDERR]))

        m = LUKS_NAME_REGEX.search(result[STDOUT])

        try:
            name = m.group(1)
        except AttributeError:
            name = None
        return name

    def get_container_device_by_name(self, name):
        ''' obtain device name based on the LUKS container name
            return None if not found
            raise ValueError if lsblk command fails
        '''
        # apparently each device can have only one LUKS container on it
        result = self._run_command([self._cryptsetup_bin, 'status', name])
        if result[RETURN_CODE] != 0:
            return None

        m = LUKS_DEVICE_REGEX.search(result[STDOUT])
        device = m.group(1)
        return device

    def is_luks(self, device):
        ''' check if the LUKS container does exist
        '''
        result = self._run_command([self._cryptsetup_bin, 'isLuks', device])
        return result[RETURN_CODE] == 0

    def run_luks_create(self, device, keyfile):
        # create a new luks container; use batch mode to auto confirm
        result = self._run_command([self._cryptsetup_bin, 'luksFormat',
                                    '-q', device, keyfile])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while creating LUKS on %s: %s'
                             % (device, result[STDERR]))

    def run_luks_open(self, device, keyfile, name):
        result = self._run_command([self._cryptsetup_bin, '--key-file', keyfile,
                                    'open', '--type', 'luks', device, name])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while opening LUKS container on %s: %s'
                             % (device, result[STDERR]))

    def run_luks_close(self, name):
        result = self._run_command([self._cryptsetup_bin, 'close', name])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while closing LUKS container %s' % (name))

    def run_luks_remove(self, device):
        wipefs_bin = self._module.get_bin_path('wipefs', True)

        name = self.get_container_name_by_device(device)
        if name is not None:
            self.run_luks_close(name)
        result = self._run_command([wipefs_bin, '--all', device])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while wiping luks container %s: %s'
                             % (device, result[STDERR]))

    def run_luks_add_key(self, device, keyfile, new_keyfile):
        ''' Add new key to given 'device'; authentication done using 'keyfile'
            Raises ValueError when command fails
        '''
        result = self._run_command([self._cryptsetup_bin, 'luksAddKey', device,
                                    new_keyfile, '--key-file', keyfile])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while adding new LUKS key to %s: %s'
                             % (device, result[STDERR]))

    def run_luks_remove_key(self, device, keyfile, force_remove_last_key=False):
        ''' Remove key from given device
            Raises ValueError when command fails
        '''
        if not force_remove_last_key:
            result = self._run_command([self._cryptsetup_bin, 'luksDump', device])
            if result[RETURN_CODE] != 0:
                raise ValueError('Error while dumping LUKS header from %s'
                                 % (device, ))
            keyslot_count = 0
            keyslot_area = False
            keyslot_re = re.compile(r'^Key Slot [0-9]+: ENABLED')
            for line in result[STDOUT].splitlines():
                if line.startswith('Keyslots:'):
                    keyslot_area = True
                elif line.startswith('  '):
                    # LUKS2 header dumps use human-readable indented output.
                    # Thus we have to look out for 'Keyslots:' and count the
                    # number of indented keyslot numbers.
                    if keyslot_area and line[2] in '0123456789':
                        keyslot_count += 1
                elif line.startswith('\t'):
                    pass
                elif keyslot_re.match(line):
                    # LUKS1 header dumps have one line per keyslot with ENABLED
                    # or DISABLED in them. We count such lines with ENABLED.
                    keyslot_count += 1
                else:
                    keyslot_area = False
            if keyslot_count < 2:
                self._module.fail_json(msg="LUKS device %s has less than two active keyslots. "
                                           "To be able to remove a key, please set "
                                           "`force_remove_last_key` to `yes`." % device)

        result = self._run_command([self._cryptsetup_bin, 'luksRemoveKey', device,
                                    '-q', '--key-file', keyfile])
        if result[RETURN_CODE] != 0:
            raise ValueError('Error while removing LUKS key from %s: %s'
                             % (device, result[STDERR]))

class ConditionsHandler(Handler):

    def __init__(self, module, crypthandler):
        super(ConditionsHandler, self).__init__(module)
        self._crypthandler = crypthandler

    def luks_create(self):
        return (self._module.params['device'] is not None and
                self._module.params['keyfile'] is not None and
                self._module.params['state'] in ('present',
                                                 'opened',
                                                 'closed') and
                not self._crypthandler.is_luks(self._module.params['device']))

    def opened_luks_name(self):
        ''' If luks is already opened, return its name.
            If 'name' parameter is specified and differs
            from obtained value, fail.
            Return None otherwise
        '''
        if self._module.params['state'] != 'opened':
            return None

        # try to obtain luks name - it may be already opened
        name = self._crypthandler.get_container_name_by_device(
            self._module.params['device'])

        if name is None:
            # container is not open
            return None

        if self._module.params['name'] is None:
            # container is already opened
            return name

        if name != self._module.params['name']:
            # the container is already open but with different name:
            # suspicious. back off
            self._module.fail_json(msg="LUKS container is already opened "
                                   "under different name '%s'." % name)

        # container is opened and the names match
        return name

    def luks_open(self):
        if (self._module.params['device'] is None or
                self._module.params['keyfile'] is None or
                self._module.params['state'] != 'opened'):
            # conditions for open not fulfilled
            return False

        name = self.opened_luks_name()

        if name is None:
            return True
        return False

    def luks_close(self):
        if ((self._module.params['name'] is None and
                self._module.params['device'] is None) or
                self._module.params['state'] != 'closed'):
            # conditions for close not fulfilled
            return False

        if self._module.params['device'] is not None:
            name = self._crypthandler.get_container_name_by_device(
                self._module.params['device'])
            # successfully getting name based on device means that luks is open
            luks_is_open = name is not None

        if self._module.params['name'] is not None:
            device = self._crypthandler.get_container_device_by_name(
                self._module.params['name'])
            # successfully getting device based on name means that luks is open
            luks_is_open = device is not None

        return luks_is_open

    def luks_add_key(self):
        if (self._module.params['device'] is None or
                self._module.params['keyfile'] is None or
                self._module.params['new_keyfile'] is None):
            # conditions for adding a key not fulfilled
            return False

        if self._module.params['state'] == 'absent':
            self._module.fail_json(msg="Contradiction in setup: Asking to "
                                   "add a key to absent LUKS.")

        return True

    def luks_remove_key(self):
        if (self._module.params['device'] is None or
                self._module.params['remove_keyfile'] is None):
            # conditions for removing a key not fulfilled
            return False

        if self._module.params['state'] == 'absent':
            self._module.fail_json(msg="Contradiction in setup: Asking to "
                                   "remove a key from absent LUKS.")

        return True

    def luks_remove(self):
        return (self._module.params['device'] is not None and
                self._module.params['state'] == 'absent' and
                self._crypthandler.is_luks(self._module.params['device']))

def run_module():
    # available arguments/parameters that a user can pass
    module_args = dict(
        state=dict(type='str', default='present', choices=['present', 'absent', 'opened', 'closed']),
        device=dict(type='str'),
        name=dict(type='str'),
        keyfile=dict(type='path'),
        new_keyfile=dict(type='path'),
        remove_keyfile=dict(type='path'),
        force_remove_last_key=dict(type='bool', default=False),
    )

    # seed the result dict in the object
    result = dict(
        changed=False,
        name=None
    )

    module = AnsibleModule(argument_spec=module_args,
                           supports_check_mode=True)

    if module.params['device'] is not None:
        try:
            statinfo = os.stat(module.params['device'])
            mode = statinfo.st_mode
            if not stat.S_ISBLK(mode) and not stat.S_ISCHR(mode):
                raise Exception('{0} is not a device'.format(module.params['device']))
        except Exception as e:
            module.fail_json(msg=str(e))

    crypt = CryptHandler(module)
    conditions = ConditionsHandler(module, crypt)

    # The conditions are in order to allow more operations in one run.
    # (e.g. create luks and add a key to it)

    # luks create
    if conditions.luks_create():
        if not module.check_mode:
            try:
                crypt.run_luks_create(module.params['device'],
                                      module.params['keyfile'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks open

    name = conditions.opened_luks_name()
    if name is not None:
        result['name'] = name

    if conditions.luks_open():
        name = module.params['name']
        if name is None:
            try:
                name = crypt.generate_luks_name(module.params['device'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        if not module.check_mode:
            try:
                crypt.run_luks_open(module.params['device'],
                                    module.params['keyfile'],
                                    name)
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['name'] = name
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks close
    if conditions.luks_close():
        if module.params['device'] is not None:
            try:
                name = crypt.get_container_name_by_device(
                    module.params['device'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        else:
            name = module.params['name']
        if not module.check_mode:
            try:
                crypt.run_luks_close(name)
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks add key
    if conditions.luks_add_key():
        if not module.check_mode:
            try:
                crypt.run_luks_add_key(module.params['device'],
                                       module.params['keyfile'],
                                       module.params['new_keyfile'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks remove key
    if conditions.luks_remove_key():
        if not module.check_mode:
            try:
                crypt.run_luks_remove_key(module.params['device'],
                                          module.params['remove_keyfile'],
                                          force_remove_last_key=module.params['force_remove_last_key'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # luks remove
    if conditions.luks_remove():
        if not module.check_mode:
            try:
                crypt.run_luks_remove(module.params['device'])
            except ValueError as e:
                module.fail_json(msg="luks_device error: %s" % e)
        result['changed'] = True
        if module.check_mode:
            module.exit_json(**result)

    # Success - return result
    module.exit_json(**result)

def main():
    run_module()

if __name__ == '__main__':
    main()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from flask import Flask, request, jsonify, make_response
from flask import render_template
from exts import db
from my_config import configs
from models import News
from sqlalchemy import and_, or_, func

app = Flask(__name__)
# app.config.from_object(configs)
app.config['SQLALCHEMY_DATABASE_URI'] = configs.SQLALCHEMY.SQLALCHEMY_DATABASE_URI
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = configs.SQLALCHEMY.SQLALCHEMY_TRACK_MODIFICATIONS
db.init_app(app)

@app.route('/')
def index():
    return render_template('index.html')

@app.route('/news/get', methods=['POST', 'GET'])
def get_news():
    count = db.session.query(func.count(News.id)).scalar()
    totalPages = (count // 10) if (count % 10 == 0) else (count // 10 + 1)

    if request.method == 'POST':
        pageNum = request.form.get("pageNum")
        sql = 'select * from news_chinese order by id limit 1,10;'
        news = db.session.execute(sql)

        return jsonify(list(news))
    else:
        pageNum = request.args.get("pageNum")
        if pageNum is None:
            pageNum = 1
            news = News.query.filter(and_(News.id.__gt__(0), News.id.__lt__(11)))
        else:
            begin = str((int(pageNum) - 1) * 10 + 1)
            sql = 'select * from news_chinese order by id limit ' + begin + ',10;'
            news = db.session.execute(sql)

        newsData = {
            'news': news,
            'totalPages': totalPages,
            'currentPage': pageNum
        }
        return render_template('historynews.html', **newsData)

if __name__ == '__main__':
    app.run()


import subprocess
import time
import logging
import logging.handlers
import threading
import re
import os

import redsocks_template

LOGGER = logging.getLogger('fqrouter.%s' % __name__)

ROOT_DIR = os.path.dirname(__file__)
LOG_DIR = '/data/data/fq.router'
REDSOCKS_LOG_FILE = os.path.join(LOG_DIR, 'redsocks.log')
REDSOCKS_LOGGER = logging.getLogger('redsocks')
handler = logging.handlers.RotatingFileHandler(
    REDSOCKS_LOG_FILE, maxBytes=1024 * 1024, backupCount=1)
handler.setFormatter(logging.Formatter('%(asctime)s %(message)s'))
REDSOCKS_LOGGER.handlers = [handler]

REFRESH_INTERVAL = 60 * 30
UPDATE_INTERVAL = 60 * 5

RE_IP_PORT = r'(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}):(\d+)'
RE_REDSOCKS_CLIENT = re.compile(RE_IP_PORT + '->')

# call back from full_proxy_service
list_proxies = None
handle_proxy_error = None
update_proxy = None
refresh_proxies = None

# internal
redsocks_process = None
updated_at = None
refreshed_at = None

def start_redsocks(proxies):
    for i in range(3):
        try:
            start_redsocks_once(proxies)
            return True
        except:
            LOGGER.exception('failed to start redsocks, retry')
            kill_redsocks()
    LOGGER.error('retry starting redsocks too many times, give up')
    return False

def start_redsocks_once(proxies):
    global redsocks_process
    if is_redsocks_live():
        LOGGER.error('another redsocks instace is running')
        return
    cfg_path = '/data/data/fq.router/redsocks.conf'
    with open(cfg_path, 'w') as f:
        f.write(redsocks_template.render(proxies))
    redsocks_process = subprocess.Popen(
        ['/data/data/fq.router/proxy-tools/redsocks', '-c', cfg_path],
        stderr=subprocess.STDOUT, stdout=subprocess.PIPE, bufsize=1, close_fds=True)
    time.sleep(2)
    if redsocks_process.poll() is None:
        LOGGER.info('redsocks seems started: %s' % redsocks_process.pid)
        t = threading.Thread(target=monitor_redsocks)
        t.daemon = True
        t.start()
    else:
        LOGGER.error('redsocks output:')
        LOGGER.error(redsocks_process.stdout.read())
        raise Exception('failed to start redsocks')

def monitor_redsocks():
    the_process = redsocks_process
    try:
        while is_redsocks_live():
            for line in iter(the_process.stdout.readline, b''):
                REDSOCKS_LOGGER.info(line.strip())
                if 'HTTP/' in line or 'No route to host' in line:
                    match = RE_REDSOCKS_CLIENT.search(line)
                    if match:
                        ip = match.group(1)
                        port = int(match.group(2))
                        for local_port, proxy in list_proxies():
                            if (ip, port) in proxy['clients']:
                                LOGGER.error(line.strip())
                                handle_proxy_error(local_port, proxy)
                update_proxies_according_to_schedule()
                refresh_proxies_according_to_schedule()
            time.sleep(1)
        LOGGER.error('redsocks died, clear proxies: %s' % the_process.poll())
        the_process.communicate()
        refresh_proxies()
    except:
        LOGGER.exception('failed to poll redsocks output')
        refresh_proxies()

def update_proxies_according_to_schedule():
    global updated_at
    if updated_at is None:
        updated_at = time.time()
    elif (time.time() - updated_at) > UPDATE_INTERVAL:
        updated_at = time.time()
        LOGGER.info('update proxies every %s seconds' % UPDATE_INTERVAL)
        for local_port, proxy in list_proxies():
            rank = int(proxy['pre_rank'] / 2) # factor in the previous performance
            LOGGER.info('update proxy %s rank: %s %s' %
                        (local_port, rank, str(proxy['connection_info'])))
            update_proxy(local_port, rank=rank, pre_rank=rank, clients=set())

def refresh_proxies_according_to_schedule():
    global refreshed_at
    if refreshed_at is None:
        refreshed_at = time.time()
    elif (time.time() - refreshed_at) > REFRESH_INTERVAL:
        refreshed_at = time.time()
        LOGGER.info('refresh proxies every %s seconds' % REFRESH_INTERVAL)
        time.sleep(15) # wait for current page
        kill_redsocks()

def kill_redsocks():
    try:
        if redsocks_process:
            LOGGER.info('found existing redsocks')
            redsocks_process.terminate()
            redsocks_process.communicate()
        LOGGER.info('redsocks killed')
    except:
        LOGGER.exception('failed to kill redsocks')
        time.sleep(2)

def is_redsocks_live():
    try:
        if not redsocks_process:
            return False
        return redsocks_process.poll() is None
    except:
        LOGGER.exception('failed to tell if redsocks is live')
        return False


#
# Copyright 2010-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved.
#
# This file is licensed under the Apache License, Version 2.0 (the "License").
# You may not use this file except in compliance with the License. A copy of
# the License is located at
#
# http://aws.amazon.com/apache2.0/
#
# This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
# CONDITIONS OF ANY KIND, either express or implied. See the License for the
# specific language governing permissions and limitations under the License.
#

# snippet-sourcedescription:[cloudcamera.py demonstrates how to mock an AWS IoT Things Graph camera device in the cloud.]
# snippet-service:[iotthingsgraph]
# snippet-keyword:[Python]
# snippet-sourcesyntax:[python]
# snippet-sourcesyntax:[python]
# snippet-keyword:[AWS IoT Things Graph]
# snippet-keyword:[Code Sample]
# snippet-sourcetype:[full-example]
# snippet-sourcedate:[2019-07-25]
# snippet-sourceauthor:[AWS]
# snippet-start:[iotthingsgraph.python.cloudcamera.complete]

from AWSIoTPythonSDK.MQTTLib import AWSIoTMQTTClient
import logging
import time
import argparse
import json

AllowedActions = ['both', 'publish', 'subscribe']

# Custom MQTT message callback
def customCallback(client, userdata, message):
    print("Received a new message: ")
    print(message.payload)
    print("from topic: ")
    print(message.topic)
    print("--------------\n\n")

# Read in command-line parameters
parser = argparse.ArgumentParser()
parser.add_argument("-e", "--endpoint", action="store", required=True, dest="host", help="Your AWS IoT custom endpoint")
parser.add_argument("-r", "--rootCA", action="store", required=True, dest="rootCAPath", help="Root CA file path")
parser.add_argument("-c", "--cert", action="store", dest="certificatePath", help="Certificate file path")
parser.add_argument("-k", "--key", action="store", dest="privateKeyPath", help="Private key file path")
parser.add_argument("-p", "--port", action="store", dest="port", type=int, help="Port number override")
parser.add_argument("-w", "--websocket", action="store_true", dest="useWebsocket", default=False,
                    help="Use MQTT over WebSocket")
parser.add_argument("-id", "--clientId", action="store", dest="clientId", default="basicPubSub",
                    help="Targeted client id")
parser.add_argument("-t", "--topic", action="store", dest="topic", default="Camera1/capture/finished", help="Targeted topic")
parser.add_argument("-m", "--mode", action="store", dest="mode", default="both",
                    help="Operation modes: %s"%str(AllowedActions))
parser.add_argument("-M", "--message", action="store", dest="message", default=True,
                    help="Message to publish")
parser.add_argument("-n", "--thingName", action="store", dest="thingName", default="Bot", help="Targeted thing name")

args = parser.parse_args()
host = args.host
rootCAPath = args.rootCAPath
certificatePath = args.certificatePath
privateKeyPath = args.privateKeyPath
port = args.port
useWebsocket = args.useWebsocket
clientId = args.clientId
topic = args.topic
thingName = args.thingName

images = ["https://images-na.ssl-images-amazon.com/images/I/41iz5Tw82IL._AC_US218_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51rMLSWgwRL._AC_US218_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/31s6UyPtjOL._AC_US218_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/41+K4pC74XL._AC_US218_.jpg"]

if args.mode not in AllowedActions:
    parser.error("Unknown --mode option %s. Must be one of %s" % (args.mode, str(AllowedActions)))
    exit(2)

if args.useWebsocket and args.certificatePath and args.privateKeyPath:
    parser.error("X.509 cert authentication and WebSocket are mutual exclusive. Please pick one.")
    exit(2)

if not args.useWebsocket and (not args.certificatePath or not args.privateKeyPath):
    parser.error("Missing credentials for authentication.")
    exit(2)

# Port defaults
if args.useWebsocket and not args.port:  # When no port override for WebSocket, default to 443
    port = 443
if not args.useWebsocket and not args.port:  # When no port override for non-WebSocket, default to 8883
    port = 8883

# Configure logging
logger = logging.getLogger("AWSIoTPythonSDK.core")
logger.setLevel(logging.DEBUG)
streamHandler = logging.StreamHandler()
formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
streamHandler.setFormatter(formatter)
logger.addHandler(streamHandler)

# Init AWSIoTMQTTClient
myAWSIoTMQTTClient = None
if useWebsocket:
    myAWSIoTMQTTClient = AWSIoTMQTTClient(clientId, useWebsocket=True)
    myAWSIoTMQTTClient.configureEndpoint(host, port)
    myAWSIoTMQTTClient.configureCredentials(rootCAPath)
else:
    myAWSIoTMQTTClient = AWSIoTMQTTClient(clientId)
    myAWSIoTMQTTClient.configureEndpoint(host, port)
    myAWSIoTMQTTClient.configureCredentials(rootCAPath, privateKeyPath, certificatePath)

# AWSIoTMQTTClient connection configuration
myAWSIoTMQTTClient.configureAutoReconnectBackoffTime(1, 32, 20)
myAWSIoTMQTTClient.configureOfflinePublishQueueing(-1)  # Infinite offline Publish queueing
myAWSIoTMQTTClient.configureDrainingFrequency(2)  # Draining: 2 Hz
myAWSIoTMQTTClient.configureConnectDisconnectTimeout(10)  # 10 sec
myAWSIoTMQTTClient.configureMQTTOperationTimeout(5)  # 5 sec

myCount = 0

# General message notification callback
def customOnMessage(message):
    print('Received message on topic %s: %s\n' % (message.topic, message.payload))
    message = {}
    global myCount
    myCount = myCount + 1
    message['lastClickedImage'] = images[myCount%4]
    messageJson = json.dumps(message)
    myAWSIoTMQTTClient.publish(thingName + "/capture/finished", messageJson, 0)
    print('Published topic %s: %s\n' % (thingName + "/capture/finished", messageJson))

myAWSIoTMQTTClient.onMessage = customOnMessage

# Connect and subscribe to AWS IoT
myAWSIoTMQTTClient.connect()
if args.mode == 'both' or args.mode == 'subscribe':
    myAWSIoTMQTTClient.subscribe(thingName + "/capture", 0, customCallback)
time.sleep(2)

# Publish to the same topic in a loop forever
loopCount = 0
while True:
    time.sleep(1)

# snippet-end:[iotthingsgraph.python.cloudcamera.complete]


#!/usr/bin/env python

# Copyright (C) 2010 Red Hat, Inc.
#
# This is free software; you can redistribute it and/or modify it
# under the terms of the GNU Lesser General Public License as
# published by the Free Software Foundation; either version 2.1 of
# the License, or (at your option) any later version.
#
# This software is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this software; if not, write to the Free
# Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
# 02110-1301 USA, or see the FSF site: http://www.fsf.org.

"""
A module containing the functions for loading the configuration
and preparing the environment for the tests.
"""
import os
import sys
import threading
import traceback
import time
import gc
import logging
import yaml
import collections
from jinja2 import Environment, FileSystemLoader

ART_CONFIG = {}
GE = {}

# garbage collector interval in seconds
GC_INTERVAL = 600

def set_cmd_line_params_in_dict(cmd_line):

    cmd_line_args = {}

    vars_to_be_treated_as_list = ['engines', 'storages']

    for param in cmd_line:
        try:
            key, value = param.split('=', 1)
        except ValueError:
            raise Exception("Expected '=' sign somewhere in '%s'." % param)
        section, var = key.split('.')
        if var in vars_to_be_treated_as_list:
            value = value.split(',')
        update_dict(cmd_line_args, {section: {var: value}})

    return cmd_line_args

def update_dict(master, update):
    for k, v in update.iteritems():
        if isinstance(v, collections.Mapping):
            r = update_dict(master.get(k, {}), v)
            master[k] = r
        else:
            master[k] = update[k]

    return master

def get_ge_yaml(cmd_line_params):
    ge_yaml = cmd_line_params.get('RUN').get('golden_environment')

    assert os.path.exists(ge_yaml)

    return ge_yaml

def generate_ge_description(ge_yaml):
    env = Environment(loader=FileSystemLoader('/'))

    runtime_yaml = 'runtime.yaml'
    template = env.get_template(ge_yaml)
    with open(ge_yaml, 'r') as f:
        context = yaml.load(f)

    rendered_yaml = template.render(context)

    with open(runtime_yaml, 'w') as f:
        f.write(rendered_yaml)

    with open(runtime_yaml, 'r') as f:
        return yaml.load(f)

    return None

def get_vds_n_passwords():

    vds_passwords = []
    vds = []

    for host in GE['hosts']:
        vds_passwords.append(host.get('password'))
        vds.append(host.get('address'))

    return vds, vds_passwords

def create_runtime_config(path_to_defaults, art_define_args):

    global ART_CONFIG
    global GE

    with open(path_to_defaults, 'r') as fh:
        defaults = yaml.load(fh)

    context = {}
    update_dict(context, defaults)

    cmd_line = set_cmd_line_params_in_dict(art_define_args)
    ge_yaml = get_ge_yaml(cmd_line)

    update_dict(context, cmd_line)

    ART_CONFIG.update(context)
    GE.update(generate_ge_description(ge_yaml))

    ART_CONFIG['DEFAULT']['PRODUCT'] = GE['product']
    ART_CONFIG['DEFAULT']['VERSION'] = GE['version']
    ART_CONFIG['REST_CONNECTION']['host'] = GE['engine_fqdn']
    ART_CONFIG['REST_CONNECTION']['uri'] = (
        ART_CONFIG['REST_CONNECTION']['uri'] % ART_CONFIG['REST_CONNECTION']
    )
    if not ART_CONFIG['REST_CONNECTION']['urisuffix']:
        ART_CONFIG['REST_CONNECTION']['uri'] = (
            ART_CONFIG['REST_CONNECTION']['uri'].replace('None', '')
        )

    vds, vds_paswords = get_vds_n_passwords()

    ART_CONFIG['PARAMETERS']['vds'] = vds
    ART_CONFIG['PARAMETERS']['vds_password'] = vds_paswords

    GE['mac_ranges'] = GE.get('mac_pools')[0].get(
        'mac_pool_ranges'
    )[0].replace(
        ',', '-'
    )

def dump_stacks(signal, frame):
    """
    In case of ART get stuck we can run kill sig command and get the
    stack traceback of each thread.
    like:
        kill -SIGUSR1 <ART PID>

    __author__ : khakimi
    :param signal: the signal number
    :type signal: int
    :param frame: the interrupted stack frame
    :type frame: frame object
    """
    id2name = dict((th.ident, th.name) for th in threading.enumerate())
    for threadId, stack in sys._current_frames().items():
        print("\nThread: {0}({1})".format(id2name[threadId], threadId))
        traceback.print_stack(f=stack)

def stuck_handler():
    """
    Check MainThread every 4 minutes if stuck.
    """
    mt = threading.current_thread().ident
    t = threading.Thread(target=stuck_check, args=(mt,))
    t.daemon = True
    t.start()

def stuck_check(main_thread):
    t = [None for i in range(5)]
    logger = logging.getLogger("stuck_handler")
    while True:
        time.sleep(240)
        t.pop(0)
        try:
            tmp = sys._current_frames()[main_thread]
        except Exception as ex:
            logger.warning(
                "sys._current_frames failed with exception: %s\n", ex
            )
            break
        t.append(traceback.format_stack(f=tmp))
        if not [x for x in t if t[0] != x]:
            logger.warn(
                "There is possiblity that MainThread is stucked. "
                "Check debug log to see traceback where it is stucked on."
            )
            logger.debug(''.join(t[-1]))

class MonitorGC(object):

    logger = logging.getLogger('art_monitor_gc')

    def __init__(self, interval=GC_INTERVAL):
        self.interval = interval
        self.thread = threading.Thread(target=self.run, name='monitor_gc')
        self.thread.daemon = True
        self.thread.start()

    def run(self):
        self.logger.info("monitor Garbage Collector started")
        saved_flags = gc.get_debug()
        gc.set_debug(0)
        try:
            while True:
                time.sleep(GC_INTERVAL)
                self.collect_gc()
        finally:
            gc.set_debug(saved_flags)
            self.logger.debug("monitor GC stopped")

    def collect_gc(self):
        try:
            collected = gc.collect()
            self.logger.debug("Collected %d objects from GC", collected)
            # Copy garbage so it is not modified while iterate over it.
            uncollectable = gc.garbage[:]
            if uncollectable:
                uncollectable = [
                    self.saferepr(obj) for obj in uncollectable
                    ]
                self.logger.warning(
                    "Found %d uncollectable objects: %s",
                    len(uncollectable), uncollectable
                )
        except Exception as exc:
            self.logger.exception("Error checking GC: %s", exc)

    def saferepr(self, obj):
        """
        Some objects from standard library fail in repr because of buggy
        __repr__ implementation. Try the builtin repr() and if it fails,
        warn and fallback to simple repr.
        """
        try:
            return repr(obj)
        except Exception as e:
            simple_repr = "<%s at 0x%x>" % (type(obj), id(obj))
            self.logger.warning("repr() failed for %s: %s", simple_repr, e)
            return simple_repr


import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
data = sns.load_dataset("iris")
#查看数据前5行
#print(data.head())
#数据预处理
#将名称简化为01234
data.columns = [0,1,2,3,4]
#将species setosa versicolor virginca 转化为1，2，3
data.loc[data[4]=='setosa',4]=1
data.loc[data[4]=='versicolor',4]=2
data.loc[data[4]=='virginica',4]=3
#构造训练train和测试text数据集==8:2
train_index=[True if i%5 != 0 else False for i in range(data.shape[0])]
text_index=[True if i%5 == 0 else False for i in range(data.shape[0])]
train=data.values[train_index,:]
text=data.values[text_index,:]
print(data)
#预览
#print(train ,'\n\n' , text)
#计算欧式距离
def distance(n1,n2):
    dist=np.sqrt(np.sum(np.power((n1-n2),2)))
    return dist
#knn实现
def knn(train,text,k):
    num=0  #记录测试正确的值
    for i in range(text.shape[0]):
        a=np.zeros(shape=(train.shape[0],2))#创建一个存放标签的矩阵
        for j in range(train.shape[0]):
            dist=distance(train[j,:-1],text[i,:-1])
            a[j,:]=dist,train[j,-1]
        df=pd.DataFrame(data=a,columns=['dist','species'])
        df=df.sort_values(['dist'])    #排序
        mode=df['species'].head(k).mode()[0]  #取众数，k为前k项
        if mode ==text[i,-1]:
            num+=1
    list1.append(num/text.shape[0])  #存储预测的准确率
list1=[]
for i in range(1,6): #k值为选取最短距离的前k项的众数
    knn(train,text,i)
print("k为1-5所对应的预测准确率为",list1)
def show():
    x=[1,2,3,4,5]
    y=list1
    plt.subplot()
    # 用来正常显示中文标签
    plt.rcParams['font.sans-serif']=['SimHei']
    plt.plot(x,y,marker='o')
    plt.xlabel('k的取值')
    plt.ylabel('正确率')
    plt.title('预测准确率随k值变化')
    for x,y in zip(x,y):
        plt.text(x,y,"(%d,%.3f)"%(x,y))
    plt.show()
show()