from django.core.management.base import AppCommand, CommandError
from django.db.models import get_models
from django.template import loader as TemplateLoader, Context
from optparse import make_option
from bricklayer.utils import ColoredOutputMixin
import os
import re
import imp

class Command(AppCommand,ColoredOutputMixin):
    help = """Creates and populates a basic admin.py file if it doesn't exists. For each model in the app,
            create an AdminModel class (if it doesn't exists) and then register the model on the admin site."""

    args = '[appname ...]'

    option_list = AppCommand.option_list + (
        make_option('--models', '-m',
                    dest='models',
                    type="string",
                    default=None,
                    help='Model/s name. Multiple models can be passed separated by a comma. E.g. -m User,Post,Comment'),
    )

    def handle_app(self, app, **options):
        app_label = app.__name__.split('.')[-2]
        app_path = os.path.dirname(app.__file__)
        admin_filename = os.path.join(os.path.dirname(app.__file__), 'admin.py')
        is_verbose = options['verbosity'] >= 1

        # Get the name of each model in the app
        app_models = [model._meta.object_name for model in get_models(app)]
        # if models
        template = TemplateLoader.get_template("admin.py.template")
        if options['models']:
            # Accept list of models as CSV
            requested_models = options["models"].split(",")
            app_models = filter(lambda model: model in requested_models, app_models)

        if not os.path.exists(admin_filename):
            self.stdout.write(app_label+".admin.py created")
            mode = "w"
            context = Context({"app": app_label,
                               "import_admin": True,
                                "models": app_models,
                                "models_to_define": app_models,
                                "models_to_register": app_models,
                                "models_to_import":app_models})

        else:
            self.stdout.write("admin.py already exists")
            """
            Check if each model is already registered.
            For each model which hasn't been registered yet,
            create an AdminClass and register the model.
            """
            admin_module = imp.load_source("admins",os.path.join(app_path, 'admin.py'))
            with open(admin_filename, 'r') as admin_file:
                admin_file_lines = admin_file.readlines()
                admin_file_text = "".join( admin_file_lines  )

            mode = 'a'
            models_to_define = []
            models_to_register = []
            models_to_import = []
            for model in app_models:
                regex = re.compile("(admin.site.register\(\s*"+model+"\s*,)",re.MULTILINE)

                #if them model wasn't loaded yet...
                if model not in dir(admin_module):
                    models_to_import.append(model)

                #if model has not been registered yet..
                if not regex.search(admin_file_text):
                    models_to_register.append(model)

                # If admin model name doesn't collide with an existing one
                if not "%sAdmin"%model in admin_file_text:
                    models_to_define.append(model)

            context = Context({"app": app_label,
                               "import_admin": "admin" in dir(admin_module),
                               "models": app_models,
                               "models_to_define": models_to_define,
                               "models_to_import":models_to_import,
                               "models_to_register": models_to_register})

        # Write admin.py
        with open(admin_filename, mode) as admin_file:
                admin_file.write(template.render(context))

        if is_verbose:
            if mode == "w":
                msg = "%s.admins created"
            else:
                msg = "%s.admins updated"
            self.print_success(msg%app_label)


# This is simple extracting text from images
# and inserted the data into a docs file

import os
import pytesseract
from PIL import Image

def get_dir_image(path):
    pathname = os.listdir(path)
    for path in pathname:
        yield path

def get_main_path(pathlist):
    for pathes in pathlist:
        yield basepath + pathes

def extract_path_from_img_name(basepath):
    path_list = list(get_dir_image(basepath))
    main_path = list(get_main_path(path_list))
    return main_path

def extract_text_from_image(basepath):
    full_path = extract_path_from_img_name(basepath)
    for full_pathes in full_path:
        img = Image.open(full_pathes)
        text = pytesseract.image_to_string(img)
        with open('image2text.docs', 'a') as wordf:
            wordf.write(text)
        print(text)

if __name__ == '__main__':
    basepath = '/home/pyking/Desktop/opencv/'
    extract_text_from_image(basepath)


### Author Name: Manqing Mao
### Luna Lander

import random
import gym
import numpy as np
from collections import deque
from keras.models import Sequential
from keras.layers import Dense
from keras import optimizers
from keras import regularizers
EPISODES = 5000

class DQNAgent:
    def __init__(self, state_size, action_size):

        # get size of state and action
        self.state_size = state_size
        self.action_size = action_size

        # These are hyper parameters for the DQN
        self.discount_factor = 0.99
        self.learning_rate = 0.001
        self.epsilon = 1.0
        self.epsilon_decay = 0.997
        self.epsilon_min = 0.101
        self.sample_size = 32
        self.train_start = 20000
        # create replay memory using deque
        self.memory = deque(maxlen=200000)
        self.train_buffer = deque(maxlen=10240)
        # create main model and target model
        self.model = self.build_model()
        self.target_model = self.build_model()

        # initialize target model
        self.update_target_model()

    # approximate Q function using Neural Network
    # state is input and Q Value of each action is output of network
    def build_model(self):
        model = Sequential()
        model.add(Dense(32, input_dim=self.state_size, activation='relu',kernel_initializer='he_uniform'))
        model.add(Dense(32, activation='relu',kernel_initializer='he_uniform'))
        model.add(Dense(32, activation='relu',kernel_initializer='he_uniform'))
        model.add(Dense(self.action_size, activation='linear',kernel_initializer='he_uniform'))
        model.summary()
        model.compile(loss='mse', optimizer=optimizers.Adam(lr=self.learning_rate))
        return model

    # after some time interval update the target model to be same with model
    def update_target_model(self):
        self.target_model.set_weights(self.model.get_weights())

    # get action from model using epsilon-greedy policy
    def get_action(self, state):
        if np.random.rand() <= self.epsilon:
            return random.randrange(self.action_size)
        else:
            q_value = self.model.predict(state)
            return np.argmax(q_value[0])

    # save sample <s,a,r,s'> to the replay memory
    def append_sample(self, state, action, reward, next_state, done):
        self.memory.append((state, action, reward, next_state, done))

    # pick samples randomly from replay memory (with batch_size)
    def train_model(self):

        #do training if buffer is full
        if len(self.train_buffer) == self.train_buffer.maxlen:
            batch_size = self.train_buffer.maxlen
            states = np.zeros((batch_size, self.state_size))
            next_states = np.zeros((batch_size, self.state_size))
            action, reward, done = [], [], []

            for i in range(batch_size):
                states[i] = self.train_buffer[i][0]
                action.append(self.train_buffer[i][1])
                reward.append(self.train_buffer[i][2])
                next_states[i] = self.train_buffer[i][3]
                done.append(self.train_buffer[i][4])

            target = self.model.predict(states)
            target_next = self.model.predict(next_states)
            target_val = self.target_model.predict(next_states)

            for i in range(batch_size):
                # Q Learning: get maximum Q value at s' from target model
                if done[i]:
                    target[i][action[i]] = reward[i]
                else:
                    a = np.argmax(target_next[i])
                    target[i][action[i]] = reward[i] + self.discount_factor * (
                        target_val[i][a])

            self.model.fit(states, target, 100, epochs=1, verbose=0)
            self.train_buffer.clear()

env = gym.make('LunarLander-v2')
state_size = env.observation_space.shape[0]
action_size = env.action_space.n
agent = DQNAgent(state_size, action_size)
scores, episodes = [], []
action_count = 0

for e in range(EPISODES):
    done = False
    score = 0
    state = env.reset()
    state = np.reshape(state, [1, state_size])

    while not done:
        action = agent.get_action(state)
        next_state, reward, done, _ = env.step(action)
        next_state = np.reshape(next_state, [1, state_size])

        agent.append_sample(state, action, reward, next_state, done)

        if len(agent.memory) > agent.train_start:
            action_count += 1

            if action_count == 4:
                mini_batch = random.sample(agent.memory, agent.sample_size)
                agent.train_buffer.extend(mini_batch)
                action_count = 0

        agent.train_model()
        score += reward
        state = next_state

        if done:
            agent.update_target_model()
            if agent.epsilon > agent.epsilon_min:
                agent.epsilon *= agent.epsilon_decay
            scores.append(score)
            episodes.append(e)
            print("episode:", e, " score:", score, " epsilon:", agent.epsilon," memory size:", len(agent.memory),"avg last 100:",np.mean(scores[-100:]))

#test model
test_scores = []

for e in range(100):
    done = False
    score = 0
    state = env.reset()
    state = np.reshape(state, [1, state_size])

    while not done:
#         env.render()
        action = np.argmax(agent.model.predict(state)[0])
        next_state, reward, done, _ = env.step(action)
        next_state = np.reshape(next_state, [1, state_size])
        score += reward
        state = next_state

        if done:
            test_scores.append(score)
            print("test episode:", e, " score:", score)

res = []
for i in range(1,len(scores)):
    j = 0 if i < 100 else i-100
    res.append(np.mean(scores[j:i]))


import cv2
import numpy as np
from scipy.stats import skew
from scipy.spatial import distance as dist
import skimage.feature

def GlobalColorHistogram(Image, channels=[0, 1, 2], n_bins=[36, 2, 2], ranges=[0, 256, 0, 256, 0, 256]):
	'''
	Calculate the global color histogram of an image
	Inputs:
		Image: Image with shape (width, height, n_channels)
		channels: Channels to compute color histogram
		n_bins: Number to quantify the channels
		ranges: Ranges for each channel
	Outputs:
		Hist: Global color histogram of the image
	'''
	Hist = cv2.calcHist([Image], channels, None, n_bins, ranges).flatten()
	Hist = Hist / np.sum(Hist)
	return Hist

def LocalColorHistogram(Image, n_rows=8, n_cols=8, channels=[0, 1, 2], n_bins=[36, 2, 2], ranges=[0, 256, 0, 256, 0, 256]):
	'''
	Calculate the local (regional) color histogram of an image
	Inputs:
		Image: Image with shape (width, height, n_channels)
		n_rows: Number of rows to cut the image
		n_cols: Number of columns to cut the image
		channels: Channels to compute color histogram
		n_bins: Number to quantify the channels
		ranges: Ranges for each channel
	Outputs:
		Hist: Local color histogram of the image
	'''
	Width, Height, Channel = Image.shape
	dW, dH = Width // n_rows, Height // n_cols
	Hist = []
	for r in range(n_rows):
		for c in range(n_cols):
			Block = Image[dW*r:dW*(r+1), dH*c:dH*(c+1)]
			Hist.append(GlobalColorHistogram(Block, channels, n_bins, ranges))
	Hist = np.array(Hist)
	return Hist

def GlobalColorMoment(Image, channels=[0, 1, 2], Weights=[1, 2, 1]):
	'''
	Calculate the global color moment of an image
	Inputs:
		Image: Image with shape (width, height, n_channels)
		channels: Channels to compute color moment
		Weights: Weight for mean, variance, skew
	Outputs:
		Moment: Global color moment of the image
	'''
	Image = Image / 255.
	Moment = []
	for i in channels:
		Channel = Image[:,:,i].flatten()
		Mean = np.mean(Channel) * Weights[0]
		Var = np.var(Channel) * Weights[1]
		Skew = skew(Channel) * Weights[2]
		Moment.append([Mean, Var, Skew])
	Moment = np.array(Moment).flatten()
	return Moment

def LocalColorMoment(Image, n_rows=8, n_cols=8, channels=[0, 1, 2], Weights=[0.01, 0.001, 3.0]):
	'''
	Calculate the local (regional) color moment of an image
	Inputs:
		Image: Image with shape (width, height, n_channels)
		n_rows: Number of rows to cut the image
		n_cols: Number of columns to cut the image
		channels: Channels to compute color moment
		Weights: Weight for mean, variance, skew
	Outputs:
		Moment: Local color moment of the image
	'''
	Width, Height, Channel = Image.shape
	dW, dH = Width // n_rows, Height // n_cols
	Moment = []
	for r in range(n_rows):
		for c in range(n_cols):
			Block = Image[dW*r:dW*(r+1), dH*c:dH*(c+1)]
			Moment.append(GlobalColorMoment(Block, channels, Weights))
	Moment = np.array(Moment)
	return Moment

def InBound(Shape, Index):
	'''
	Check if the Index is in bound of image
	Inputs:
		Shape: Shape of the image
		Index: The index to check
	Outputs:
		Valid: True id index is in bound, otherwize False
	'''
	if (Index[0] >= 0) and (Index[1] >= 0) and (Index[0] < Shape[0]) and (Index[1] < Shape[1]):
		Valid = True
	else:
		Valid = False
	return Valid

def GetNeighborColors(Image, Center, Distance):
	'''
	Get the color of neighbors that are certain distance away from center
	Inputs:
		Image: Image after quantization
		Center: The query center index
		Distance: The desired distance for neighbors
	Outputs:
		Colors: Colors of valid neighbors
	'''
	Shape = Image.shape
	X, Y = Center
	Colors = []
	P1 = [X-Distance, Y]
	if InBound(Shape, P1):
		Colors.append(Image[P1[0]][P1[1]])
	P2 = [X+Distance, Y]
	if InBound(Shape, P2):
		Colors.append(Image[P2[0]][P2[1]])
	P3 = [X, Y-Distance]
	if InBound(Shape, P3):
		Colors.append(Image[P3[0]][P3[1]])
	P4 = [X, Y+Distance]
	if InBound(Shape, P4):
		Colors.append(Image[P4[0]][P4[1]])
	P5 = [X-Distance, Y-Distance]
	if InBound(Shape, P5):
		Colors.append(Image[P5[0]][P5[1]])
	P6 = [X-Distance, Y+Distance]
	if InBound(Shape, P6):
		Colors.append(Image[P6[0]][P6[1]])
	P7 = [X+Distance, Y-Distance]
	if InBound(Shape, P7):
		Colors.append(Image[P7[0]][P7[1]])
	P8 = [X+Distance, Y+Distance]
	if InBound(Shape, P8):
		Colors.append(Image[P8[0]][P8[1]])
	return Colors

def GetProb(Image, Center, Distance):
	'''
	Get the probability of pixels that are in certain distance away has the same color as the center
	Inputs:
		Image: Image after quantization
		Center: The query center index
		Distance: The desired distance for neighbors
	Outputs:
		Prob: Probability of neighbor color equal to center color
	'''
	Shape = Image.shape
	X, Y = Center
	TargetColor = Image[X][Y]
	NeighborColors = np.array(GetNeighborColors(Image, Center, Distance))
	Prob = np.mean(NeighborColors == TargetColor)
	return Prob

def AutoCorrelogram(Image, channels=[0, 1, 2], n_bins=[16, 2, 2], ranges=[0, 180, 0, 256, 0, 256], Distances=[1, 2, 4, 8, 16, 32, 64]):
	'''
	Calculate the auto-correlogram of an image
	Inputs:
		Image: Image with shape (width, height, n_channels)
		channels: Channels to compute auto-correlogram
		n_bins: Number of bins for each channel
		ranges: Range of value of each channel
		Distances: Distances to compute auto-correlogram
	Outputs:
		Correlogram: The auto-correlogram
	'''
	W, H, C = Image.shape
	temp = []
	for i in channels:
		Channel = np.array(Image[:, :, i])
		Channel -= ranges[2*i]
		Channel = Channel // ((ranges[2*i+1] - ranges[2*i]) / n_bins[i])
		temp.append(Channel)
	temp = np.array(temp).astype(int)
	Image = np.zeros((W, H), int)
	Mult = 1
	for i in range(temp.shape[0]):
		Image += temp[i, :, :] * Mult
		Mult *= n_bins[i]
	Correlogram = []
	for d in Distances:
		ColorProbs = [[] for _ in range(np.prod(n_bins))]
		for w in range(W):
			for h in range(H):
				ColorProbs[Image[w, h]].append(GetProb(Image, [w, h], d))
		ColorProbs = [(sum(x) / len(x)) if len(x) > 0 else 1e-3 for x in ColorProbs]
		Correlogram.append(ColorProbs)
	Correlogram = np.array(Correlogram).flatten()
	return Correlogram

def GaborFilteredImages(Image, Kernels=[16], Sigmas=[1.0, 2.0, 4.0, 8.0], Thetas=[0, np.pi/8, np.pi/4, np.pi*3/8, np.pi/2, np.pi*5/8, np.pi*3/4, np.pi*7/8]):
	'''
	Generate images after gabor filters
	Inputs:
		Image: Gray scale image
		Kernels: Kernel sizes for gabor filters
		Sigmas: Sigmas for gabor filters
		Thetas: Orientation of the filters
	Outputs:
		FilteredImages: Images after gabor filters
	'''
	Image = Image.squeeze()
	GaborFilters = []
	for K in Kernels:
		for t in Thetas:
			for s in Sigmas:
				Filter = cv2.getGaborKernel((K, K), s, t, np.pi/2, 0.5, 0, ktype=cv2.CV_32F)
				GaborFilters.append(Filter / np.sum(Filter))
	FilteredImages = []
	for F in GaborFilters:
		FilteredImage = cv2.filter2D(Image, cv2.CV_8UC3, F)
		FilteredImages.append(np.expand_dims(FilteredImage, axis=-1))
	FilteredImages = np.array(FilteredImages)
	return FilteredImages

def ExtractGaborFeature(GaborImages):
	'''
	Extract features from gabor filter filtered images
	Imputs:
		GaborImages: List of images after gabor filters
	Outputs:
		Features: Features extracted from images
	'''
	Features = []
	for i in GaborImages:
		i = i.squeeze()
		i = i / 255.
		LocalEnergy = np.mean(np.square(i))
		MeanAmplitude = np.mean(i)
		VarAmplitude = np.var(i)
		Features.append(np.array([LocalEnergy, MeanAmplitude, VarAmplitude]))
	Features = np.array(Features).T
	return Features

def GaborGlobalHistogram(GaborImages, channels=[0], n_bins=[32], ranges=[0, 256]):
	'''
	Calculate the global histogram of gabor images
	Inputs:
		GaborImages: List of images after gabor filters
		channels: Channels to compute color histogram
		n_bins: Number to quantify the channels
		ranges: Ranges for each channel
	Outputs:
		GaborHist: List of global histogram of the gabor images
	'''
	GaborHist = []
	for Image in GaborImages:
		Hist = GlobalColorHistogram(Image, channels=channels, n_bins=n_bins, ranges=ranges)
		GaborHist.append(Hist)
	GaborHist = np.array(GaborHist)
	return GaborHist

def GaborLocalHistogram(Images, n_rows=8, n_cols=8, channels=[0], n_bins=[32], ranges=[0, 256]):
	'''
	Calculate the local (regional) histogram of gabor images
	Inputs:
		GaborImages: List of images after gabor filters
		n_rows: Number of rows to cut the image
		n_cols: Number of columns to cut the image
		channels: Channels to compute color histogram
		n_bins: Number to quantify the channels
		ranges: Ranges for each channel
	Outputs:
		GaborHist: List of local histogram of the gabor images
	'''
	GaborImages = GaborFilteredImages(Images)
	GaborHist = []
	for Image in GaborImages:
		Hist = LocalColorHistogram(Image, n_rows, n_cols, channels, n_bins, ranges)
		GaborHist.append(Hist)
	GaborHist = np.array(GaborHist).reshape(len(GaborImages) * n_rows * n_cols, -1)
	return GaborHist

def LocalBinaryPattern(Image, radius=3, points=64):
	'''
	Calculate local binary pattern histogram
	Inputs:
		Image: Gray scale image
		radius: RRadius of circle (spatial resolution of the operator)
		points: Number of circularly symmetric neighbour set points (quantization of the angular space)
	Output:
		Hist: The local binary pattern histogram
	'''
	Image = Image.squeeze()
	LBP = np.expand_dims(skimage.feature.local_binary_pattern(Image, points, radius, method='uniform'), axis=-1).astype(np.float32)
	Hist = cv2.calcHist([LBP], [0], None, [points+2], [0, points+2]).flatten()
	Hist = Hist / np.sum(Hist)
	return Hist

def GridLocalBinaryPattern(Image, n_rows=8, n_cols=8, radius=3, points=64):
	'''
	Calculate the local (regional) color histogram of an image
	Inputs:
		Image: Gray scale image
		n_rows: Number of rows to cut the image
		n_cols: Number of columns to cut the image
		radius: Radius of circle (spatial resolution of the operator)
		points: Number of circularly symmetric neighbour set points (quantization of the angular space)
	Outputs:
		Hist: Grid local color histogram of the image
	'''
	Width, Height, Channel = Image.shape
	dW, dH = Width // n_rows, Height // n_cols
	Hist = []
	for r in range(n_rows):
		for c in range(n_cols):
			Block = Image[dW*r:dW*(r+1), dH*c:dH*(c+1)]
			Hist.append(LocalBinaryPattern(Block, radius, points))
	Hist = np.array(Hist)
	return Hist

def HistogramofOrientedGradients(Image, n_rows=8, n_cols=8, orientations=64):
	'''
	Calculate the histogram of oriented gradients (HOG) an image
	Inputs:
		Image: Gray scale image
		n_rows: Number of rows to cut the image
		n_cols: Number of columns to cut the image
		orientaions: Number of orientations
	Outputs:
		HOG: Histogram of oriented gradients of the image
	'''
	Image = Image.squeeze()
	Width, Height = Image.shape
	dW, dH = Width // n_rows, Height // n_cols
	HOG = skimage.feature.hog(Image, orientations=orientations, pixels_per_cell=(dW, dH), cells_per_block=(1, 1), feature_vector=False)
	HOG = np.reshape(HOG, (-1, orientations))
	GlobalGradient = np.sum(HOG, axis=0).reshape(1, -1)
	return HOG

def PyramidHOG(Image, n_rows=[1, 2, 4, 8, 16, 32], n_cols=[1, 2, 4, 8, 16, 32], orientations=64):
	'''
	Calculate the HOG of an image in different resolutions
	Inputs:
		Image: Gray scale image
		n_rows: Number of rows to cut the image
		n_cols: Number of columns to cut the image
		orientaions: Number of orientations
	Outputs:
		PyramidHOG: Pyramid histogram of oriented gradients of the image
	'''
	Image = Image.squeeze()
	PyramidHOG_ = []
	for i in range(len(n_rows)):
		HOG = HistogramofOrientedGradients(Image, n_rows[i], n_cols[i], orientations) * (1 / n_rows[i]**2)
		PyramidHOG_.append(HOG)
	PyramidHOG = PyramidHOG_[0]
	for PHOG in PyramidHOG_[1:]:
		PyramidHOG = np.concatenate((PyramidHOG, PHOG), axis=0)
	return PyramidHOG

def ShapeIndex(Image, Sigmas=[0.5, 1, 2, 4, 8, 16]):
	'''
	Calculate the image index of an image with different sigmas, then calculate histogram
	Inputs:
		Image: Gray scale image
		Sigmas: List of sigmas
	Outputs:
		ShapeIndexHist: The shape index histogram
	'''
	Image = Image.squeeze()
	ShapeIndexHist = []
	for s in Sigmas:
		ShapeIndex = np.expand_dims(skimage.feature.shape_index(Image, sigma=s), axis=-1).astype(np.float32)
		# Flat regions are difined as nan
		ShapeIndex[np.isnan(ShapeIndex)] = 1.3
		Hist = GlobalColorHistogram(ShapeIndex, channels=[0], n_bins=[10], ranges=[-1.125, 1.375])
		# Since falt regions are mostly background, we give a small weight to it
		Hist[-1] *= 0.1
		Hist = Hist / np.sum(Hist)
		ShapeIndexHist.append(Hist)
	ShapeIndexHist = np.array(ShapeIndexHist)
	return ShapeIndexHist

def GetSIFT(Image, limit=None):
	'''
	Detect and compute SIFT descriptors of an image
	Inputs:
		Image: Gray scale image
		limit: Maximun number of keypoints to detect
	Outputs:
		des: The keypoints SIFT descriptors
	'''
	# Use default sigma(1.6) would find 0 keypoints in a tennis image QAQ
	if limit is None:
		sift = cv2.xfeatures2d.SIFT_create()
	else:
		sift = cv2.xfeatures2d.SIFT_create(limit)
	kp, des = sift.detectAndCompute(Image, None)
	return des

def GetDenseSIFT(Image, n_rows=32, n_cols=32):
	'''
	Compute SIFT discriptors on grids
	Inputs:
		Image: Gray scale image
		n_rows: Number of rows to cut the image
		n_cols: Number of columns to cut the image
	Outputs:
		des: The grid SIFT descriptors
	'''
	sift = cv2.xfeatures2d.SIFT_create()
	Width, Height, Channel = Image.shape
	dW, dH = Width // n_rows, Height // n_cols
	KPs = []
	for x in range(0, Width, dW):
		for y in range(0, Height, dH):
			kp = cv2.KeyPoint(x, y, (dW+dH)/2)
			KPs.append(kp)
	kp, des = sift.compute(Image, KPs)
	return des

def GetPyramidSIFT(Image, n_rows=[1, 2, 4, 8, 16, 32, 64], n_cols=[1, 2, 4, 8, 16, 32, 64]):
	'''
	Compute SIFT discriptors on different sizes of grids
	Inputs:
		Image: Gray scale image
		n_rows: List of number of rows to cut the image
		n_cols: List of number of columns to cut the image
	Outputs:
		Pyramiddes: The pyramid grid SIFT descriptors
	'''
	Pyramiddes_ = []
	for i in range(len(n_rows)):
		des = GetDenseSIFT(Image, n_rows=n_rows[i], n_cols=n_cols[i])
		Pyramiddes_.append(des)
	Pyramiddes = Pyramiddes_[0]
	for des in Pyramiddes_[1:]:
		Pyramiddes = np.concatenate((Pyramiddes, des), axis=0)
	return Pyramiddes


#!/usr/bin/env python
# ROS node libs
import rospy
from std_msgs.msg import String
from sensor_msgs.msg import Image,PointCloud2
from std_msgs.msg import Int16, Float32MultiArray
from multiprocessing import Queue, Pool
from cv_bridge import CvBridge, CvBridgeError

# General libs
import numpy as np
import os
import sys
import tensorflow as tf
import cv2
# import PIL
import time
import time
import datetime
# Detector libs
# from object_detection.utils import ops as utils_ops
# from object_detection.utils import label_map_util
# from object_detection.utils import visualization_utils as vis_util

# GPU settings: Select GPUs to use. Coment it to let the system decide
#os.environ["CUDA_VISIBLE_DEVICES"]="0"

class ros_tensorflow_obj():
    def __init__(self):
        # ## Initial msg
        rospy.loginfo('  ## Starting ROS Tensorflow interface ##')
        # ## Load a (frozen) Tensorflow model into memory.
        detection_graph = tf.Graph()
        with detection_graph.as_default():
            od_graph_def = tf.GraphDef()

            # with tf.gfile.GFile("./object_detection/saved_model_frcnn_4000_samples_00/frozen_inference_graph.pb", 'rb') as fid:
            # with tf.gfile.GFile("./object_detection/mayank_pb/frozen_inference_graph_100000.pb", 'rb') as fid:
            # with tf.gfile.GFile("./object_detection/mayank_pb/frozen_inference_graph_14134.pb", 'rb') as fid:
            with tf.gfile.GFile("./object_detection/mayank_pb/frozen_inference_graph_30812.pb", 'rb') as fid:
            # with tf.gfile.GFile("./object_detection/mayank_pb/frozen_inference_graph_new.pb", 'rb') as fid:
                serialized_graph = fid.read()
                od_graph_def.ParseFromString(serialized_graph)
                tf.import_graph_def(od_graph_def, name='')

        # ## Loading label map
        # Label maps map indices to category names, so that when our convolution network predicts `5`, we know that this corresponds to `airplane`.  Here we use internal utility functions, but anything 			that returns a dictionary mapping integers to appropriate string labels would be fine
        # self.category_index = label_map_util.create_category_index_from_labelmap("./object_detection/Baidu_models/Baidu_ssd_model_0/haval_label_map.pbtxt", use_display_name=True)
        # self.category_index = label_map_util.create_category_index_from_labelmap("./object_detection/mayank_pb/bdd_traffic_label_map_simgle.pbtxt", use_display_name=True)
        # ## Get Tensors to run from model
        self.image_tensor = detection_graph.get_tensor_by_name('image_tensor:0')
        # Each box represents a part of the image where a particular object was detected.
        self.boxes = detection_graph.get_tensor_by_name('detection_boxes:0')
        # Each score represent how level of confidence for each of the objects.
        # Score is shown on the result image, together with the class label.
        self.scores = detection_graph.get_tensor_by_name('detection_scores:0')
        self.classes = detection_graph.get_tensor_by_name('detection_classes:0')
        self.num_detections = detection_graph.get_tensor_by_name('num_detections:0')

        # # Tensorflow Session opening: Creates a session with log_device_placement set to True.
        # ## Session configuration
        config=tf.ConfigProto(log_device_placement=True)
        config.log_device_placement = True
        config.gpu_options.allow_growth = True

        # ## Session openning
        try:
            with detection_graph.as_default():
                self.sess = tf.Session(graph=detection_graph, config = config)
                rospy.loginfo('  ## Tensorflow session open: Starting inference... ##')
        except ValueError:
            rospy.logerr('   ## Error when openning session. Please restart the node ##')
            rospy.logerr(ValueError)

        # image_np= cv2.imread("baid_0.jpg",1)
        # image_np_expanded = np.expand_dims(image_np, axis=0)
        # (boxes_out, scores_out, classes_out, num_detections_out) = self.sess.run(
        # [self.boxes, self.scores, self.classes, self.num_detections], feed_dict={self.image_tensor: image_np_expanded})
#############################################33333
        # loading keras graph
        global mymodel
        mymodel = tf.keras.models.load_model("./saved_models/new_model/model_b_lr-4_ep150_ba32_1.h")
        # this is key : save the graph after loading the model
        global graph
        graph = tf.get_default_graph()
        # self.mymodel = tf.keras.models.load_model("./saved_models/new_model/model_b_lr-4_ep150_ba32.h")
        # print("My model is:", mymodel.summary())
        # self.light_co=[]
        #####################################################
        print("ready to process----------------------------------------------------------")
        # [print(n.name) for n in tf.get_default_graph().as_graph_def().node]
        # ROS environment setup
        # ##  Define subscribers
        self.subscribers_def()
        # print("subs")
        # ## Define publishers
        self.publishers_def()
        # print("pub")
        # ## Get cv_bridge: CvBridge is an object that converts between OpenCV Images and ROS Image messages
        self._cv_bridge = CvBridge()
        self.now = rospy.Time.now()

    # Define subscribers
    def subscribers_def(self):
        subs_topic = 'crop_tl_image'
        self._sub = rospy.Subscriber( subs_topic , Image, self.img_callback, queue_size=1, buff_size=2**24)
        # self._sub = rospy.Subscriber( subs_topic , Image, self.img_callback, queue_size=1, buff_size=100)

    # Define publishers
    def publishers_def(self):
        tl_bbox_topic = '/tl_bbox_topic_megs'
        self._pub = rospy.Publisher('tl_bbox_topic', Float32MultiArray, queue_size=1)

    # Camera image callback
    def img_callback(self, image_msg):
        image_msg.encoding = "bgr8"
        first = time.time()
        image_np = self._cv_bridge.imgmsg_to_cv2(image_msg, "bgr8")
        # image_np = cv2.cvtColor(image_np, cv2.COLOR_BGR2RGB)

        rows, cols, channels = image_np.shape
        image_np_expanded = np.expand_dims(image_np, axis=0)
        second = time.time()
        time_cost = round((second-first)*1000)
        print("post processing Time",time_cost)
        third = time.time()
        (boxes_out, scores_out, classes_out, num_detections_out) = self.sess.run([self.boxes, self.scores, self.classes, self.num_detections],feed_dict={self.image_tensor: image_np_expanded})
        numboxes = np.squeeze(num_detections_out)
        # print("num_detection are",num_detections_out)
        numboxes = 1

        # print("confidence score are",scores_out[:, 0:8])
        tl_bbox = Float32MultiArray()
        if int(numboxes)>=1:
            tmp = -np.ones(7*int(numboxes)+1)

            for i in range(0,int(numboxes)):
                score = float(np.squeeze(scores_out)[i])
                bbox = [float(v) for v in np.squeeze(boxes_out)[i]]
                tmp[0] = numboxes
                if score > 0.3:
                    x_top_left = bbox[1] * cols
                    y_top_left = bbox[0] * rows
                    x_bottom_right = bbox[3] * cols
                    y_bottom_right = bbox[2] * rows
                    width = x_bottom_right - x_top_left
                    height = y_bottom_right - y_top_left
                    tmp[5*i+1] = x_top_left
                    tmp[5*i+2] = y_top_left
                    tmp[5*i+3] = width
                    tmp[5*i+4] = height
                    tmp[5*i+5] = score
                    # tmp[6] = 1
                    # tmp[7] = .999

                    #########################################################333
                    # Adding color model into the tensorflow model
                    # # traffic_light = ["red", "green", "black"]
                    traffic_light =  ["black", "red", "yellow", "green"]

                    # mymodel = tf.keras.models.load_model("./saved_models/new_model/model_b_lr-4_ep150_ba32.h")
                    # frame = img[int(ymin):int(ymax), int(xmin):int(xmax)]
                    frame = image_np[int(y_top_left):int(y_bottom_right), int(x_top_left):int(x_bottom_right)]
                    # introdusing the color model here
                    # test_image = frame.resize((32, 32))
                    test_image = cv2.resize(frame, (32, 32))
                    # test_image = image
                    test_image = tf.keras.preprocessing.image.img_to_array(test_image)
                    test_image = np.expand_dims(test_image, axis=0)
                    test_image /= 255
                    # print("My model is:" ,mymodel.summary())
                    # amodel=self.mymodel
                    with graph.as_default():
                        result = mymodel.predict(test_image)
                        # print("result is ", result)
                    # print("Prediction done")
                    # print(result)
                    #
                    tmp[6] = int(result.argmax())
                    tmp[7 * i + 7] =0.999
                    # tmp[7] = float(result[0, result.argmax()])

                    light_color = traffic_light[result.argmax()]
                    #
                    # light_color = light_color+"\n"
                    # selfight_co.append(light_color)
                    # with open('somefile4.txt', 'a') as the_file:
                    #     the_file.write(light_color)
                    # # '_______________________________________'

                    # cv2.imshow('images', frame)
                    # cv2.waitKey(1)
                    # cv2.destroyAllWindows()
                    # # cv2.waitKey(10)
                    # top = (int(x_top_left), int(y_top_left))
                    # bottom = (int(x_bottom_right), int(y_bottom_right))
                    # cv2.rectangle(image_np, pt1=top, pt2=bottom, color=(0, 255, 0), thickness=2)
                    # cv2.putText(image_np, light_color, (250, 90), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0),lineType=cv2.LINE_AA)
                    ################################################################
        else:
            tmp = [-1.0, 10.0, 10.0, 10.0, 10.0,10.0,10.0]

        # cv2.imshow("abcd", image_np)
        # cv2.waitKey(1)
        # output_path="./image5/"
        # ts = time.time()
        # st = datetime.datetime.fromtimestamp(ts).strftime('%f')
        # cut_frame = output_path
        # cut_frame = cut_frame + "image_" + str(st) + ".jpg"
        # print(cut_frame)
        # cv2.imwrite(cut_frame, image_np)

        tl_bbox.data = tmp
        thorth = time.time()
        time_cost = round((thorth-third)*1000)
        print("Inference Time",time_cost)
        print("bbox ",tl_bbox.data)
        self._pub.publish(tl_bbox)
# Spin once
def spin(self):
    rospy.spin()

def main():
    rospy.init_node('tf_object_detector', anonymous=True)
    tf_ob = ros_tensorflow_obj()
    try:
        rospy.spin()
    except KeyboardInterrupt:
        print("Shutting down")
if __name__ == '__main__':
    main()


import sys
import pandas
import sqlalchemy
import sqlite3

class validater:
    def __init__(self, db):
        self.db = db

    def compare_volume(self):
        connection = sqlite3.connect(self.db)
        cursor0 = connection.cursor()
        cursor1 = connection.cursor()

        #log
        query1 = "select symdev, emu, dev_id, cyl from `VMAX_symdev_log_table`"
        #dws
        query2 = "select symdev, Emu, Device_ID, cyl from `VMAX_symdev_dws_table`"

        query_output1 = cursor0.execute(query1)
        output1 = query_output1.fetchall()
        query_output2 = cursor1.execute(query2)
        output2 = query_output2.fetchall()

        for index, i in enumerate(output2):
            if ":" in i[0]:
                print(i)
                alu_start = i[0].split(":")[0]
                alu_start_list = list(alu_start)
                if alu_start_list[0] == "0":
                    count_start = 0
                    count_end = 1
                    if alu_start_list[1] == "0":
                        count_end += 1
                        if alu_start_list[2] == "0":
                            count_end += 1
                            if alu_start_list[3] == "0":
                                count_end += 1
                del alu_start_list[count_start:count_end]
                alu_start = "".join(alu_start_list)
                alu_start = int(alu_start, 16)

                alu_end = i[0].split(":")[1]
                alu_end_list = list(alu_end)
                if alu_end_list[0] == "0":
                    count_start = 0
                    count_end = 1
                    if alu_end_list[1] == "0":
                        count_end += 1
                        if alu_end_list[2] == "0":
                            count_end += 1
                            if alu_end_list[3] == "0":
                                count_end += 1
                del alu_end_list[count_start:count_end]
                alu_end = "".join(alu_end_list)
                alu_end = int(alu_end, 16)
                emu = i[1]
                Device_ID = i[2]
                cyl	= i[3]

                alu_start_abso = format(str(hex(alu_start)).split("0x")[1].upper(), '0>5')
                output2[index] = (alu_start_abso, emu, Device_ID, cyl)
                # print((alu_start_abso, emu, Device_ID, cyl))
                alu_diff = alu_end - alu_start
                for diff_a in range(alu_diff):
                    alu_med = alu_start + diff_a + 1
                    alu_med = format(str(hex(alu_med)).split("0x")[1].upper(), '0>5')
                    output2.append((alu_med, emu, Device_ID, cyl))
                    # print((alu_med, emu, Device_ID, cyl))

        set1 = set(output1)
        set2 = set(output2)

        compare1 = set1.difference(set2) #item in the log but not in the dws
        compare2 = set2.difference(set1) #item in the dws but not in the log

        result = ""
        message = ""

        if compare1:
            if compare2:
                result = "the differences in in both DWS and Log"
                message = "Please check details."
            else:
                result = "the difference: {}".format(compare1)
                message = "This pamareter is not in DWS. there is an possibility that you conducted the unnecessary settings."
        elif compare2:
            result = "the difference: {}".format(compare2)
            message = "This parameters are not in log. You should check the configuration what you set."
        else:
            result = "complete"
            message = "Your work is done completely."

        return result, message

    def compare_mv(self):
        connection = sqlite3.connect(self.db)
        cursor0 = connection.cursor()
        cursor1 = connection.cursor()

        #masking
        '''
        compare the MV_name
        '''

        #log
        query0 = "select distinct MV_name from `VMAX_Masking_log_table` where 'MV_name' is not null"
        #dws
        query1 = "select distinct MV_name from `VMAX_Masking_dws_table` where 'MV_name' is not null and 'MV_name' != 'None'"

        query_output0 = cursor0.execute(query0)
        output0 = query_output0.fetchall()
        query_output1 = cursor1.execute(query1)
        output1 = query_output1.fetchall()

        set0 = set(output0)
        set1 = set(output1)
        if (None,) in set0:
            set0 = set0 - set((None,))
        if (None,) in set1:
            set0 = set1 - set((None,))

        compare0 = set0.difference(set1) #item in the log but not in the dws
        compare1 = set1.difference(set0) #item in the dws but not in the log

        if compare0 or compare1:
            message_mv = "Masking view: there is differences in MV name. "
        else:
            message_mv = "Masking view: complete"

        #IG
        '''
        compare the set of MV_name and IG_name
        '''

        cursor2 = connection.cursor()
        cursor3 = connection.cursor()

        #log
        query2 = "select distinct MV_name, IG_name from `VMAX_Masking_log_table` WHERE 'MV_name' IS NOT NULL and 'IG_name' IS NOT NULL"
        #dws
        query3 = "select distinct MV_name, IG_name from `VMAX_Masking_dws_table` WHERE 'MV_name' IS NOT NULL and 'IG_name' IS NOT NULL"

        query_output2 = cursor2.execute(query2)
        output2 = query_output2.fetchall()
        query_output3 = cursor3.execute(query3)
        output3 = query_output3.fetchall()

        drop_box_2 = []
        for i, output in enumerate(output2):
            if "None" in str(output[0]):
                drop_box_2.append(output)
            elif "None" in str(output[1]):
                drop_box_2.append(output)

        drop_box_3 = []
        for i, output in enumerate(output3):
            if "None" in str(output[0]):
                drop_box_3.append(output)
            elif "None" in str(output[1]):
                drop_box_3.append(output)

        if drop_box_2:
            output2 = [x for x in output2 if x not in drop_box_2]
        if drop_box_3:
            output3 = [x for x in output3 if x not in drop_box_3]

        set2 = set(output2)
        set3 = set(output3)

        compare2 = set2.difference(set3) #item in the log but not in the dws
        compare3 = set3.difference(set2) #item in the dws but not in the log

        if compare2 or compare3:
            message_ig = "IG name: there is differences in IG name. "
        else:
            message_ig = "IG name: complete"

        '''
        compare the set of IG_name, flag and consistent lun
        '''
        cursor4 = connection.cursor()
        cursor5 = connection.cursor()

        #log
        query4 = "select distinct IG_name, flag, Consistent_lun from `VMAX_Masking_log_table` where flag is not null and Consistent_lun is not null"
        #dws
        query5 = "select distinct IG_name, flag, Consistent_lun from `VMAX_Masking_dws_table` where flag is not null and Consistent_lun is not null"

        query_output4 = cursor4.execute(query4)
        output4 = query_output4.fetchall()
        query_output5 = cursor5.execute(query5)
        output5 = query_output5.fetchall()

        drop_box_4 = []
        for i, output in enumerate(output4):
            if "None" in str(output[0]):
                drop_box_4.append(output)
            elif "None" in str(output[1]):
                drop_box_4.append(output)

        drop_box_5 = []
        for i, output in enumerate(output5):
            if "None" in str(output[0]):
                drop_box_5.append(output)
            elif "None" in str(output[1]):
                drop_box_5.append(output)

        if drop_box_4:
            output4 = [x for x in output4 if x not in drop_box_4]
        if drop_box_5:
            output5 = [x for x in output5 if x not in drop_box_5]

        set4 = set(output4)
        set5 = set(output5)

        compare4 = set4.difference(set5) #item in the log but not in the dws
        compare5 = set5.difference(set4) #item in the dws but not in the log

        if compare4 or compare5:
            message_ig_op = "IG option: there is differences in IG option. "
        else:
            message_ig_op = "IG option: complete"

        '''
        compare the set of IG_name, flag and consistent lun
        '''
        cursor6 = connection.cursor()
        cursor7 = connection.cursor()

        #log
        query6 = "select distinct IG_name, WWN, Alias, Cascade_IG from `VMAX_Masking_log_table` WHERE 'IG_name' IS NOT NULL and 'WWN' IS NOT NULL and 'Alias' IS NOT NULL"
        #dws
        query7 = "select distinct IG_name, WWN, Alias, Cascade_IG from `VMAX_Masking_dws_table` WHERE 'IG_name' IS NOT NULL and 'WWN' IS NOT NULL and 'Alias' IS NOT NULL"

        query_output6 = cursor6.execute(query6)
        output6 = query_output6.fetchall()
        query_output7 = cursor7.execute(query7)
        output7 = query_output7.fetchall()

        drop_box_6 = []
        for i, output in enumerate(output6):
            if "None" in str(output[0]):
                drop_box_6.append(output)
            elif "None" in str(output[1]):
                drop_box_6.append(output)

        drop_box_7 = []
        for i, output in enumerate(output7):
            if "None" in str(output[0]):
                drop_box_7.append(output)
            elif "None" in str(output[1]):
                drop_box_7.append(output)

        if drop_box_6:
            output6 = [x for x in output6 if x not in drop_box_6]
        if drop_box_7:
            output7 = [x for x in output7 if x not in drop_box_7]

        set6 = set(output6)
        set7 = set(output7)

        compare6 = set6.difference(set7) #item in the log but not in the dws
        compare7 = set7.difference(set6) #item in the dws but not in the log

        if compare6 or compare7:
            message_ig_par = "IG paremeters: there is differences in IG parameters. "
        else:
            message_ig_par = "IG paremeters: complete"

        #PG
        '''
        compare the set of MV_name and PG_name
        '''

        cursor8 = connection.cursor()
        cursor9 = connection.cursor()

        #log
        query8 = "select distinct MV_name, PG_name from `VMAX_Masking_log_table` WHERE 'MV_name' IS NOT NULL and 'PG_name' IS NOT NULL"
        #dws
        query9 = "select distinct MV_name, PG_name from `VMAX_Masking_dws_table` WHERE 'MV_name' IS NOT NULL and 'PG_name' IS NOT NULL"

        query_output8 = cursor8.execute(query8)
        output8 = query_output8.fetchall()
        query_output9 = cursor9.execute(query9)
        output9 = query_output9.fetchall()

        drop_box_8 = []
        for i, output in enumerate(output8):
            if "None" in str(output[0]):
                drop_box_8.append(output)
            elif "None" in str(output[1]):
                drop_box_8.append(output)

        drop_box_9 = []
        for i, output in enumerate(output9):
            if "None" in str(output[0]):
                drop_box_9.append(output)
            elif "None" in str(output[1]):
                drop_box_9.append(output)

        if drop_box_8:
            output8 = [x for x in output8 if x not in drop_box_8]
        if drop_box_9:
            output9 = [x for x in output9 if x not in drop_box_9]

        set8 = set(output8)
        set9 = set(output9)

        compare8 = set8.difference(set9) #item in the log but not in the dws
        compare9 = set9.difference(set8) #item in the dws but not in the log

        if compare8 or compare9:
            message_pg = "PG name: there is differences in PG name. "
        else:
            message_pg = "PG name: complete"

        '''
        compare the set of PG_name, dir and port
        '''

        cursor10 = connection.cursor()
        cursor11 = connection.cursor()

        #log
        query10 = "select PG_name, dir, port FROM `VMAX_Masking_log_table` where dir is not null and port is not null"
        #dws
        query11 = "select PG_name, dir, port FROM `VMAX_Masking_dws_table` where dir is not null and port is not null"

        query_output10 = cursor10.execute(query10)
        output10 = query_output10.fetchall()
        query_output11 = cursor11.execute(query11)
        output11 = query_output11.fetchall()

        drop_box_10 = []
        for i, output in enumerate(output10):
            if "None" in str(output[0]):
                drop_box_10.append(output)
            elif "None" in str(output[1]):
                drop_box_10.append(output)

        drop_box_11 = []
        for i, output in enumerate(output11):
            if "None" in str(output[0]):
                drop_box_11.append(output)
            elif "None" in str(output[1]):
                drop_box_11.append(output)

        if drop_box_10:
            output10 = [x for x in output10 if x not in drop_box_10]
        if drop_box_11:
            output11 = [x for x in output11 if x not in drop_box_11]

        set10 = set(output10)
        set11 = set(output11)

        compare10 = set10.difference(set11) #item in the log but not in the dws
        compare11 = set11.difference(set10) #item in the dws but not in the log

        if compare10 or compare11:
            message_pg_op = "PG parameters: there is differences in PG parameters. "
        else:
            message_pg_op = "PG parameters: complete"

        #SG
        '''
        compare the set of MV_name and SG_name
        '''

        cursor12 = connection.cursor()
        cursor13 = connection.cursor()

        #log
        query12 = "select distinct MV_name, SG_name from `VMAX_Masking_log_table` WHERE 'MV_name' IS NOT NULL and 'SG_name' IS NOT NULL"
        #dws
        query13 = "select distinct MV_name, SG_name from `VMAX_Masking_dws_table` WHERE 'MV_name' IS NOT NULL and 'SG_name' IS NOT NULL"

        query_output12 = cursor12.execute(query12)
        output12 = query_output12.fetchall()
        query_output13 = cursor13.execute(query13)
        output13 = query_output13.fetchall()

        drop_box_12 = []
        for i, output in enumerate(output12):
            if "None" in str(output[0]):
                drop_box_12.append(output)
            elif "None" in str(output[1]):
                drop_box_12.append(output)

        drop_box_13 = []
        for i, output in enumerate(output13):
            if "None" in str(output[0]):
                drop_box_13.append(output)
            elif "None" in str(output[1]):
                drop_box_13.append(output)

        if drop_box_12:
            output12 = [x for x in output12 if x not in drop_box_12]
        if drop_box_13:
            output13 = [x for x in output13 if x not in drop_box_13]

        set12 = set(output12)
        set13 = set(output13)

        compare12 = set12.difference(set13) #item in the log but not in the dws
        compare13 = set13.difference(set12) #item in the dws but not in the log

        if compare12 or compare13:
            message_sg = "SG name: there is differences in SG name. "
        else:
            message_sg = "SG name: complete"

        '''
        compare the set of SG_name and Compression
        '''

        cursor14 = connection.cursor()
        cursor15 = connection.cursor()

        #log
        query14 = "select distinct SG_name, Compression from `VMAX_Masking_log_table` where Compression is not null"
        #dws
        query15 = "select distinct SG_name, Compression from `VMAX_Masking_dws_table` where Compression is not null"

        query_output14 = cursor14.execute(query14)
        output14 = query_output14.fetchall()
        query_output15 = cursor15.execute(query15)
        output15 = query_output15.fetchall()

        drop_box_14 = []
        for i, output in enumerate(output14):
            if "None" in str(output[0]):
                drop_box_14.append(output)
            elif "None" in str(output[1]):
                drop_box_14.append(output)

        drop_box_15 = []
        for i, output in enumerate(output15):
            if "None" in str(output[0]):
                drop_box_15.append(output)
            elif "None" in str(output[1]):
                drop_box_15.append(output)

        if drop_box_14:
            output14 = [x for x in output14 if x not in drop_box_14]
        if drop_box_15:
            output15 = [x for x in output15 if x not in drop_box_15]

        set14 = set(output14)
        set15 = set(output15)

        compare14 = set14.difference(set15) #item in the log but not in the dws
        compare15 = set15.difference(set14) #item in the dws but not in the log

        if compare14 or compare15:
            message_sg_op = "SG option: there is differences in SG compression. "
        else:
            message_sg_op = "SG option: complete"

        '''
        compare the set of SG_name, symdev and hlu
        '''

        cursor16 = connection.cursor()
        cursor17 = connection.cursor()

        #log
        query16 = "select SG_name, Cascade_SG, symdev, hlu from `VMAX_Masking_log_table` where symdev is not null"
        #dws
        query17 = "select SG_name, Cascade_SG, symdev, hlu from `VMAX_Masking_dws_table` where symdev is not null"

        query_output16 = cursor16.execute(query16)
        output16 = query_output16.fetchall()
        query_output17 = cursor17.execute(query17)
        output17 = query_output17.fetchall()

        for index, i in enumerate(output17):
            if ":" in i[1]:
                sg_name = i[0]

                alu_start = i[1].split(":")[0]
                alu_start_list = list(alu_start)
                if alu_start_list[0] == "0":
                    count_start = 0
                    count_end = 1
                    if alu_start_list[1] == "0":
                        count_end += 1
                        if alu_start_list[2] == "0":
                            count_end += 1
                            if alu_start_list[3] == "0":
                                count_end += 1
                del alu_start_list[count_start:count_end]
                alu_start = "".join(alu_start_list)
                alu_start = int(alu_start, 16)

                alu_end = i[1].split(":")[1]
                alu_end_list = list(alu_end)
                if alu_end_list[0] == "0":
                    count_start = 0
                    count_end = 1
                    if alu_end_list[1] == "0":
                        count_end += 1
                        if alu_end_list[2] == "0":
                            count_end += 1
                            if alu_end_list[3] == "0":
                                count_end += 1
                del alu_end_list[count_start:count_end]
                alu_end = "".join(alu_end_list)
                alu_end = int(alu_end, 16)

                hlu_start = i[2].split("-")[0]
                hlu_start_list = list(hlu_start)
                if hlu_start_list[0] == "0":
                    count_start = 0
                    count_end = 1
                    if hlu_start_list[1] == "0":
                        count_end += 1
                del hlu_start_list[count_start:count_end]
                hlu_start = "".join(hlu_start_list)
                hlu_start = int(hlu_start, 16)

                hlu_end = i[2].split("-")[1]
                hlu_end_list = list(hlu_end)
                if hlu_end_list[0] == "0":
                    count_start = 0
                    count_end = 1
                    if hlu_end_list[1] == "0":
                        count_end += 1
                del hlu_end_list[count_start:count_end]
                hlu_end = "".join(hlu_end_list)
                hlu_end = int(hlu_end, 16)

                alu_start_abso = format(str(hex(alu_start)).split("0x")[1].upper(), '0>5')
                hlu_start_abso = format(str(hex(hlu_start)).split("0x")[1].upper(), '0>3')
                output17[index] = (sg_name, alu_start_abso, hlu_start_abso)
                alu_diff = alu_end - alu_start
                for diff_a in range(alu_diff):
                    alu_med = alu_start + diff_a + 1
                    alu_med = format(str(hex(alu_med)).split("0x")[1].upper(), '0>5')
                    hlu_med = hlu_start + diff_a + 1
                    hlu_med = format(str(hex(hlu_med)).split("0x")[1].upper(), '0>3')
                    output17.append((sg_name, alu_med, hlu_med))

        drop_box_16 = []
        for i, output in enumerate(output16):
            if "None" in str(output[0]):
                drop_box_16.append(output)
            elif "None" in str(output[1]):
                drop_box_16.append(output)

        drop_box_17 = []
        for i, output in enumerate(output17):
            if "None" in str(output[0]):
                drop_box_17.append(output)
            elif "None" in str(output[1]):
                drop_box_17.append(output)

        if drop_box_16:
            output16 = [x for x in output16 if x not in drop_box_16]
        if drop_box_17:
            output17 = [x for x in output17 if x not in drop_box_17]

        set16 = set(output16)
        set17 = set(output17)

        compare16 = set16.difference(set17) #item in the log but not in the dws
        compare17 = set17.difference(set16) #item in the dws but not in the log

        if compare16 or compare17:
            message_sg_par = "SG parameters: there is differences in SG parameters."
        else:
            message_sg_par = "SG parameters: complete"

        all_message = [message_mv, message_ig, message_ig_op, message_ig_par, message_pg, message_pg_op, message_sg, message_sg_op, message_sg_par]

        for i in all_message:
            print(i)

        result = ""

        count = 0
        for mess in all_message:
            if "complete" in mess:
                continue
            else:
                count += 1

        if count == 0:
            result = "complete"
        else:
            result = "miss"

        return result, all_message

def main():
    dbfile = 'db.sqlite3'
    log_validater = validater(dbfile)

    dev_result, dev_message = log_validater.compare_volume()

    #mv
    conf_result, conf_message = log_validater.compare_mv()
    print(conf_result)

    return dev_result, dev_message, conf_result, conf_message

if __name__ == "__main__":
    main()