#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Sat Aug 19 13:56:24 2017

@author: jack
"""

import itertools
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image

def plot_images(images, cls_true=None, cls_pred=None, errors_por=None, size_cm=(10,10)):
    assert len(images) == 9

    plt.figure(figsize=(size_cm))
    fig, axes = plt.subplots(3, 3)
    fig.subplots_adjust(hspace=.25, wspace=.25)

    for i, ax in enumerate(axes.flat):
        ax.imshow(images[i], cmap='binary')

        if cls_true is not None:
            if cls_pred is None:
                xlabel = "T: {0}".format(cls_true[i])
            else:
                xlabel = "T: {0}, P: {1}, Per: {2:>.1%}".format(cls_true[i], cls_pred[i],errors_por[i])

        ax.set_xlabel(xlabel)

        ax.set_xticks([])
        ax.set_yticks([])

    plt.show()

##############################################################################################################################################

def plot_rest_images(images, cls_true=None, cls_pred=None, errors_por=None, size_cm=(1.2,1.2), cmap=None, axis=False, colorbar=False):
    for i in range(len(images)):
        fig = plt.figure(figsize=size_cm)

        if cmap:
            cax = plt.imshow(images[i], cmap=cmap)
        else:
            cax = plt.imshow(images[i])
        if not axis:
            plt.axis('off')
        if colorbar:
            fig.colorbar(cax)
        plt.show()
        if cls_true is not None:
            if cls_pred is None:
                print("T: {0}".format(cls_true[i]))
            else:
                print("T: {0}, P: {1}, Per: {2:>.1%}".format(cls_true[i], cls_pred[i],errors_por[i]))

##############################################################################################################################################

def plot_32_im(images,
               errors_por=None,
               size_cm=[20,20],
               cmap=None,
               axis=False,
               colorbar=False,
               ):
    if len(images.shape) == 4:
        images = np.reshape(images, [images.shape[1],images.shape[2],images.shape[3]])
    _large = 100
    blink_im = []
    for _i in range(_large):
        blink_im.append(np.eye(_large)[_i])
    blink_im = np.reshape(blink_im, [_large,_large])
    blink_im = (blink_im + list(reversed(blink_im))) > 0

    if axis:
        space = .95
    else:
        space = .2
    if size_cm[0]>=30:
        space = .4

    fig, axes = plt.subplots(4, 8, figsize=size_cm)
    fig.subplots_adjust(hspace=space, wspace=space)

    for i, ax in enumerate(axes.flat):
        try:
            if cmap:
                cax = ax.imshow(images[:,:,i], cmap=cmap)
            else:
                cax = ax.imshow(images[:,:,i])
            _exist_im = True
        except:
            cax = ax.imshow(blink_im, cmap=cmap)
            _exist_im = False
        if not axis or not _exist_im:
            ax.axis('off')
        if colorbar and _exist_im:
            fig.colorbar(cax, ax = ax)

    plt.show()

##############################################################################################################################################

def plot_confusion_matrix(cm, classes,
                      normalize=False,
                      title='Confusion matrix',
                      cmap=plt.cm.Blues):
    """
    This function prints and plots the confusion matrix.
    Normalization can be applied by setting `normalize=True`.
    """
    plt.imshow(cm, interpolation='nearest', cmap=cmap)
    plt.title(title)
    plt.colorbar()
    tick_marks = np.arange(len(classes))
    plt.xticks(tick_marks, classes, rotation=45)
    plt.yticks(tick_marks, classes)

    if normalize:
        cm = cm.astype('float') / cm.sum(axis=1)[:, np.newaxis]
        print("Normalized confusion matrix")
    else:
        print('Confusion matrix')

    thresh = cm.max() / 2.
    for i, j in itertools.product(range(cm.shape[0]), range(cm.shape[1])):
        plt.text(j, i, cm[i, j],
                 horizontalalignment="center",
                 color="white" if cm[i, j] > thresh else "black")

    plt.tight_layout()
    plt.ylabel('True label')
    plt.xlabel('Predicted label')
    plt.show()

##############################################################################################################################################

def predictCustom(model, X, threshold, default_class=0):
    """
    Return the predict results of given non time-series network.

    Parameters
    ----------
    sess : TensorFlow session
        sess = tf.InteractiveSession()
    network : a TensorLayer layer
        the network will be trained
    X : numpy array
        the input
    x : placeholder
        for inputs
    y_op : placeholder
        the argmax expression of softmax outputs

    Examples
    --------
    >>> see tutorial_mnist_simple.py
    >>> y = network.outputs
    >>> y_op_soft = tf.nn.softmax(y) # this
    >>> y_op = tf.argmax(y_op_soft, 1)
    >>> print(data.predictCustom(sess, network, X_test, x, y_op))
    """
    if len(X.shape) == 3:
        X = np.reshape(X, [-1, X.shape[0], X.shape[1], X.shape[2]])
    predict = model.predict(X)
    respuesta = []
    for pred in predict:
        if np.argmax(pred) == default_class:
            respuesta.append(default_class)
        else:
            if pred[np.argmax(pred)] >= threshold:
                respuesta.append(np.argmax(pred))
            else:
                respuesta.append(default_class)
    respuesta = np.asarray(respuesta)
    return respuesta

##############################################################################################################################################

def evaluation(y_test=None, y_predict=None, n_classes=None, Print=True):
    """
    Input the predicted results, targets results and
    the number of class, return the confusion matrix, F1-score of each class,
    accuracy and macro F1-score.

    Parameters
    ----------
    y_test : numpy.array or list
        target results
    y_predict : numpy.array or list
        predicted results
    n_classes : int
        number of classes

    Examples
    --------
    >>> c_mat, f1, acc, f1_macro = evaluation(y_test, y_predict, n_classes)
    """
    from sklearn.metrics import confusion_matrix, f1_score, accuracy_score
    acc   = accuracy_score(y_test, y_predict)
    c_mat = confusion_matrix(y_test, y_predict, labels = [x for x in range(n_classes)])
    f1    = f1_score(y_test, y_predict, average = None, labels = [x for x in range(n_classes)])
    f1_macro = f1_score(y_test, y_predict, average='macro')
    if Print:
        print('confusion matrix: \n',c_mat)
        print('f1-score:',f1)
        print('f1-score(macro):',f1_macro)   # same output with > f1_score(y_true, y_pred, average='macro')
        print('accuracy-score:', acc)
    return c_mat, f1, acc, f1_macro

##############################################################################################################################################

def norm(im, n):
    if n is None:
        return n
    elif n == 0:
        return np.divide(im,255)
    elif n == 1:
        _range = [0,1]
        return ((im - np.min(im)) * (np.max(_range) - np.min(_range)) / (np.max(im) - np.min(im))) + np.min(im)
    else:
        raise Exception("No existe ese tipo de normalizacion/nTipos:/n/tNone = Sin normalizacion/n/t0 = Se divide en 255/n/t1 = Normalizacion local de la imagen")

##############################################################################################################################################

def predict(model, X):
    """
    Return the predict results of given non time-series network.

    Parameters
    ----------
    sess : TensorFlow session
        sess = tf.InteractiveSession()
    network : a TensorLayer layer
        the network will be trained
    X : numpy array
        the input
    x : placeholder
        for inputs
    y_op : placeholder
        the argmax expression of softmax outputs

    Examples
    --------
    >>> see tutorial_mnist_simple.py
    >>> y = network.outputs
    >>> y_op = tf.argmax(tf.nn.softmax(y), 1)
    >>> print(tl.utils.predict(sess, network, X_test, x, y_op))
    """
    return model.predit(X)

##############################################################################################################################################

def str_to_im_resize(im, img_resize, filters):
    """
    Function to return from a string to a resized image
    """
    if filters == 'NONE':
        return im.resize((img_resize, img_resize), Image.NONE)
    elif filters == 'BICUBIC':
        return im.resize((img_resize, img_resize), Image.BICUBIC)
    elif filters == 'NEAREST':
        return im.resize((img_resize, img_resize), Image.NEAREST)
    elif filters == 'BOX':
        return im.resize((img_resize, img_resize), Image.BOX)
    elif filters == 'BILINEAR':
        return im.resize((img_resize, img_resize), Image.BILINEAR)
    elif filters == 'HAMMING':
        return im.resize((img_resize, img_resize), Image.HAMMING)
    elif filters == 'LANCZOS':
        return im.resize((img_resize, img_resize), Image.LANCZOS)
    elif filters == 'ANTIALIAS':
        return im.resize((img_resize, img_resize), Image.ANTIALIAS)
    else:
        raise Exception("Filter doesn't exist")


import asyncio, time

def timeit(func):
    """ Timing decorator for coroutines """
    async def process(func, *args, **params):
        if asyncio.iscoroutinefunction(func):
            print('this function is a coroutine: {}'.format(func.__name__))
            return await func(*args, **params)
        else:
            print('this is not a coroutine')
            return func(*args, **params)

    async def helper(*args, **params):
        print('{}.time'.format(func.__name__))
        start = time.time()
        result = await process(func, *args, **params)

        # Test normal function route...
        # result = await process(lambda *a, **p: print(*a, **p), *args, **params)

        print('>>>', time.time() - start)
        return result

    return helper


# coding=UTF8
# encoding=utf8

# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#		 http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from __future__ import absolute_import
from __future__ import print_function
# import tensorflow.python.platform
import collections
import math
import numpy as np
# import os
import random
# from six.moves import urllib
from six.moves import xrange	# pylint: disable=redefined-builtin
import tensorflow as tf
import codecs
import jieba.analyse
import sys
import re
import sklearn.preprocessing
from numpy import array
import chroma
# import matplotlib.pyplot as plt
# import matplotlib.colors as colors
# import matplotlib.cm as cmx

import sqlite3

reload(sys)
sys.setdefaultencoding('utf8')

#const
top_n_articles = 20
vocabulary_size = 3000 #top n words to be trained
#num_steps = 10001 #training steps, ex. 100001
num_steps = 100001
plot_only = 500 #ploting data pts
output_model = "model/tars-mode-c-v4"
output_filename = "img/tsne-c4.png"

# Read the data into a string.
def read_data():
	jieba.set_dictionary('dict.txt.big')
	stop1 = u"[，。、「」（）\(\)\.【】『』：；・．～？＝＼／！＠\@＄\$％＆\&\%\-\/\\\>\<\~\:\,\[\]\?\!\=\+＋\*＊]*"
	stop2 = "http[a-zA-Z\.\:\/\-\?\#]*"
	stop3 = "[0-9\.]*"
	result = []

	word_cate = {}
	color_dict = {}

	'''
	#read color dict
	header = True
	f = codecs.open('category_color.csv', 'r', encoding='utf8')
	for row in f:
		if(header):
			header = False
			continue
		row = row.replace('"', '').split(',')
		color_dict[row[0]] = None

	arr = color_dict.keys()
	cNorm  = colors.Normalize(vmin=-0.1, vmax=1)
	scalarMap = cmx.ScalarMappable(norm=cNorm, cmap=plt.get_cmap('nipy_spectral') )
	for i in range(len(arr)):
		color = scalarMap.to_rgba(i/float(len(arr)))
		color_dict[arr[i]] = chroma.Color(color, format="RGB")
		print(arr[i]+', '+str(color_dict[arr[i]]))
	'''

	#read article
	conn = sqlite3.connect('ptt_course.db')
	c = conn.cursor()

	for i, article in enumerate(c.execute('SELECT Content FROM Article_Directory WHERE Category = "評價" AND Content not null')):
		print('reading %i' % i)
		result += ['UNK']
		text = article[0].replace('  ', '')
		text = re.sub(stop1, "", text)
		text = re.sub(stop2, "_URL_", text)
		text = re.sub(stop3, "", text)
		seg_list = jieba.lcut(text, cut_all=False)
		word_list = []
		for seg in seg_list:
			if(len(seg) == 1):
				continue
			word_list.append(seg)
		result += word_list
	'''
	for partition in range(0, 10000, 1000):
		count = 0
		filename = 'article_v2_%s_%s.arff' % (str(partition), str(partition+1000))
		print(filename)
		f = codecs.open(filename, 'r', encoding='utf8')
		for row in f:
			if(row[0]=='@'): continue
			#if(count > 100): break
			if(count > top_n_articles): break
			print("T: "+row.split(',')[4].replace('"', ''))
			temp_cate_list = row.split(',')[1].replace('"', '').split(';')
			cate_list = []
			for cate in temp_cate_list:
				if cate in color_dict.keys():
					cate_list.append(cate)
			text = row.split(',')[5].replace('"', '')
			#print(text)
			text = re.sub(stop1, "", text)
			text = re.sub(stop2, "_URL_", text)
			text = re.sub(stop3, "", text)
			seg_list = jieba.lcut(text, cut_all=False)
			word_list = []
			for seg in seg_list:
				if(len(seg) == 1):
					continue
				word_list.append(seg)
			#print("/ ".join(seg_list))

			score_list = jieba.analyse.textrank(text, topK=20, withWeight=True, allowPOS=('ns', 'n', 'vn', 'v'))
			#here to normalize
			key_word_list = []
			norm_score_list = []

			for word, score in score_list:
				if (len(word) == 1):
					continue
				key_word_list.append(word)
				norm_score_list.append([score])

			a = array(norm_score_list, dtype='f')
			if(len(norm_score_list)>0):
				norm_score_list = sklearn.preprocessing.normalize(a, axis=0)

			for i in range(len(score_list)):
				word = key_word_list[i]
				score = norm_score_list[i][0]
				for cate in cate_list:
					try:
						word_cate[word][cate] += [score]
					except:
						try:
							word_cate[word][cate] = [score]
						except:
							word_cate[word] = {}
							word_cate[word][cate] = [score]

			result = result + word_list
			count += 1

	word_color = {}
	for word, cate_list in word_cate.iteritems():

		color_list = []
		for cate, score_list in cate_list.iteritems():
			score = sum(score_list) / float(len(score_list))
			#color = chroma.Color(color_dict[cate])
			color = color_dict[cate]
			color.alpha = score
			color_list.append(color)

		word_color[word] = chroma.Color('#000000')
		for color in color_list:
			word_color[word] += color

		if(word_color[word]==chroma.Color('#000000')):
			word_color[word] = chroma.Color('#FFFFFF')
	'''

	return result

words = read_data()
print('Data size', len(words))
# Step 2: Build the dictionary and replace rare words with UNK token.
#vocabulary_size = 1000

def build_dataset(words):
	count = [['UNK', -1]]
	count.extend(collections.Counter(words).most_common(vocabulary_size - 1))
	dictionary = dict()
	for word, _ in count:
		dictionary[word] = len(dictionary) - 1
	data = list()
	unk_count = 0
	for word in words:
		if word in dictionary:
			index = dictionary[word]
		else:
			index = 0	# dictionary['UNK']
			unk_count = unk_count + 1
		data.append(index)
	count[0][1] = unk_count
	reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys()))
	return data, count, dictionary, reverse_dictionary
data, count, dictionary, reverse_dictionary = build_dataset(words)
del words	# Hint to reduce memory.
print('Most common words (+UNK)', count[:10])
print('Sample data', data[:20])
data_index = 0
# Step 4: Function to generate a training batch for the skip-gram model.
def generate_batch(batch_size, num_skips, skip_window):
	global data_index
	assert batch_size % num_skips == 0
	assert num_skips <= 2 * skip_window
	batch = np.ndarray(shape=(batch_size), dtype=np.int32)
	labels = np.ndarray(shape=(batch_size, 1), dtype=np.int32)
	span = 2 * skip_window + 1 # [ skip_window target skip_window ]
	buffer = collections.deque(maxlen=span)
	for _ in range(span):
		buffer.append(data[data_index])
		data_index = (data_index + 1) % len(data)
	for i in range(batch_size // num_skips):
		target = skip_window	# target label at the center of the buffer
		targets_to_avoid = [ skip_window ]
		for j in range(num_skips):
			while target in targets_to_avoid:
				target = random.randint(0, span - 1)
			targets_to_avoid.append(target)
			batch[i * num_skips + j] = buffer[skip_window]
			labels[i * num_skips + j, 0] = buffer[target]
		buffer.append(data[data_index])
		data_index = (data_index + 1) % len(data)
	return batch, labels
batch, labels = generate_batch(batch_size=8, num_skips=2, skip_window=1)
for i in range(8):
	print(batch[i], '->', labels[i, 0])
	print(reverse_dictionary[batch[i]])
	print(reverse_dictionary[batch[i]], '->', reverse_dictionary[labels[i, 0]])
# Step 5: Build and train a skip-gram model.
batch_size = 128
embedding_size = 128	# Dimension of the embedding vector.
skip_window = 1			 # How many words to consider left and right.
num_skips = 2				 # How many times to reuse an input to generate a label.
# We pick a random validation set to sample nearest neighbors. Here we limit the
# validation samples to the words that have a low numeric ID, which by
# construction are also the most frequent.
valid_size = 16		 # Random set of words to evaluate similarity on.
valid_window = 100	# Only pick dev samples in the head of the distribution.
valid_examples = np.array(random.sample(np.arange(valid_window), valid_size))
num_sampled = 64		# Number of negative examples to sample.
graph = tf.Graph()
with graph.as_default():
	# Input data.
	train_inputs = tf.placeholder(tf.int32, shape=[batch_size])
	train_labels = tf.placeholder(tf.int32, shape=[batch_size, 1])
	valid_dataset = tf.constant(valid_examples, dtype=tf.int32)
	# Ops and variables pinned to the CPU because of missing GPU implementation
	with tf.device('/gpu:0'):
		# Look up embeddings for inputs.
		embeddings = tf.Variable(
				tf.random_uniform([vocabulary_size, embedding_size], -1.0, 1.0))
		embed = tf.nn.embedding_lookup(embeddings, train_inputs)
		# Construct the variables for the NCE loss
		nce_weights = tf.Variable(
				tf.truncated_normal([vocabulary_size, embedding_size],
														stddev=1.0 / math.sqrt(embedding_size)))
		nce_biases = tf.Variable(tf.zeros([vocabulary_size]))
	# Compute the average NCE loss for the batch.
	# tf.nce_loss automatically draws a new sample of the negative labels each
	# time we evaluate the loss.
	loss = tf.reduce_mean(
			tf.nn.nce_loss(nce_weights, nce_biases, embed, train_labels,
										 num_sampled, vocabulary_size))
	# Construct the SGD optimizer using a learning rate of 1.0.
	optimizer = tf.train.GradientDescentOptimizer(1.0).minimize(loss)
	# Compute the cosine similarity between minibatch examples and all embeddings.
	norm = tf.sqrt(tf.reduce_sum(tf.square(embeddings), 1, keep_dims=True))
	normalized_embeddings = embeddings / norm
	valid_embeddings = tf.nn.embedding_lookup(
			normalized_embeddings, valid_dataset)
	similarity = tf.matmul(
			valid_embeddings, normalized_embeddings, transpose_b=True)
# Step 6: Begin training
#num_steps = 100001

with tf.Session(graph=graph) as session:
	# We must initialize all variables before we use them.
	tf.initialize_all_variables().run()
	saver = tf.train.Saver()  # defaults to saving all variables - in this case w and b
	print("Initialized")
	average_loss = 0
	for step in xrange(num_steps):
		batch_inputs, batch_labels = generate_batch(
				batch_size, num_skips, skip_window)
		feed_dict = {train_inputs : batch_inputs, train_labels : batch_labels}
		# We perform one update step by evaluating the optimizer op (including it
		# in the list of returned values for session.run()
		_, loss_val = session.run([optimizer, loss], feed_dict=feed_dict)
		average_loss += loss_val
		if step % 2000 == 0:
			if step > 0:
				average_loss = average_loss / 2000
			# The average loss is an estimate of the loss over the last 2000 batches.
			print("Average loss at step ", step, ": ", average_loss)
			saver.save(session, output_model, global_step=step)
			average_loss = 0
		# note that this is expensive (~20% slowdown if computed every 500 steps)
		if step % 10000 == 0:
			sim = similarity.eval()
			for i in xrange(valid_size):
				valid_word = reverse_dictionary[valid_examples[i]]
				top_k = 8 # number of nearest neighbors
				nearest = (-sim[i, :]).argsort()[1:top_k+1]
				log_str = "Nearest to %s:" % valid_word
				for k in xrange(top_k):
					close_word = reverse_dictionary[nearest[k]]
					log_str = "%s %s," % (log_str, close_word)
				print(log_str)
	final_embeddings = normalized_embeddings.eval()
# Step 7: Visualize the embeddings.
def plot_with_labels(low_dim_embs, labels, filename=output_filename):
	assert low_dim_embs.shape[0] >= len(labels), "More labels than embeddings"
	font = FontProperties(fname="NotoSansCJKtc-Medium.otf", size=14)
	plt.figure(figsize=(32, 32))	#in inches
	plt.axis([-30, 30, -30, 30])
	for i, label in enumerate(labels):
		x, y = low_dim_embs[i,:]
		kwcolor = 'black'
		plt.scatter(x, y)
		plt.annotate(label,
								 xy=(x, y),
								 xytext=(5, 2),
								 textcoords='offset points',
								 ha='right',
								 va='bottom',
								 fontproperties=font,
								 color = kwcolor)
	'''
	count = 0
	for word, color in color_dict.iteritems():
		plt.text(-29, 25-1*count, word, ha='left', va='bottom', fontproperties=font, color=str(color)[:7])
		count += 1
	'''
	plt.savefig(filename)
try:
	from sklearn.manifold import TSNE
	from matplotlib.font_manager import FontProperties
	import matplotlib
	matplotlib.use('Agg') #no display
	import matplotlib.pyplot as plt
	tsne = TSNE(perplexity=30, n_components=2, init='pca', n_iter=5000)
	low_dim_embs = tsne.fit_transform(final_embeddings[:plot_only,:])
	labels = [reverse_dictionary[i] for i in xrange(plot_only)]
	plot_with_labels(low_dim_embs, labels)
except ImportError:
	print("Please install sklearn and matplotlib to visualize embeddings.")


#!/usr/bin/env python
# Copyright 2013 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

from fuelmenu.common import dialog
from fuelmenu.common.modulehelper import ModuleHelper
from fuelmenu.common import replace
import fuelmenu.common.urwidwrapper as widget
from fuelmenu.settings import Settings
import logging
import netaddr
import os
import re
import socket
import subprocess
import urwid
import urwid.raw_display
import urwid.web_display
log = logging.getLogger('fuelmenu.mirrors')
blank = urwid.Divider()

class dnsandhostname(urwid.WidgetWrap):
    def __init__(self, parent):
        self.name = "DNS & Hostname"
        self.priority = 50
        self.visible = True
        self.netsettings = dict()
        self.deployment = "pre"
        self.getNetwork()
        self.gateway = self.get_default_gateway_linux()
        self.extdhcp = True
        self.parent = parent

        #UI Text
        self.header_content = ["DNS and hostname setup", "Note: Leave "
                               "External DNS blank if you do not have "
                               "Internet access."]
        self.fields = ["HOSTNAME", "DNS_DOMAIN", "DNS_SEARCH", "DNS_UPSTREAM",
                       "blank", "TEST_DNS"]
        hostname, sep, domain = os.uname()[1].partition('.')
        self.defaults = \
            {
                "HOSTNAME": {"label": "Hostname",
                             "tooltip": "Hostname to use for Fuel master node",
                             "value": hostname},
                "DNS_UPSTREAM": {"label": "External DNS",
                                 "tooltip": "DNS server(s) (comma separated) \
to handle DNS requests (example 8.8.8.8)",
                                 "value": "8.8.8.8"},
                "DNS_DOMAIN": {"label": "Domain",
                               "tooltip": "Domain suffix to user for all \
nodes in your cluster",
                               "value": domain},
                "DNS_SEARCH": {"label": "Search Domain",
                               "tooltip": "Domains to search when looking up \
DNS (space separated)",
                               "value": domain},
                "TEST_DNS": {"label": "Hostname to test DNS:",
                             "value": "www.google.com",
                             "tooltip": "DNS record to resolve to see if DNS \
is accessible"}
            }

        self.oldsettings = self.load()
        self.screen = None
        self.fixEtcHosts()

    def fixEtcHosts(self):
        #replace ip for env variable HOSTNAME in /etc/hosts
        if self.netsettings[self.parent.managediface]["addr"] != "":
            managediface_ip = self.netsettings[
                self.parent.managediface]["addr"]
        else:
            managediface_ip = "127.0.0.1"
        found = False
        with open("/etc/hosts") as fh:
            for line in fh:
                if re.match("%s.*%s" % (managediface_ip,
                            socket.gethostname()), line):
                    found = True
                    break
        if not found:
            expr = ".*%s.*" % socket.gethostname()
            replace.replaceInFile("/etc/hosts", expr, "%s   %s %s" % (
                                  managediface_ip,
                                  socket.gethostname(),
                                  socket.gethostname().split('.')[0]))

    def setEtcResolv(self, nameserver="default"):
        if nameserver == "default":
            nss = self.defaults['DNS_UPSTREAM']['value'].split(',')
        else:
            nss = [nameserver]
        with open("/etc/resolv.conf", "w") as fh:
            for ns in nss:
                fh.write("nameserver %s\n" % ns)

    def check(self, args):
        """Validate that all fields have valid values through sanity checks."""
        self.parent.footer.set_text("Checking data...")
        self.parent.refreshScreen()
        #Get field information
        responses = dict()

        for index, fieldname in enumerate(self.fields):
            if fieldname == "blank":
                pass
            else:
                responses[fieldname] = self.edits[index].get_edit_text()

        ###Validate each field
        errors = []

        #hostname must be under 60 chars
        if len(responses["HOSTNAME"]) >= 60:
            errors.append("Hostname must be under 60 chars.")

        #hostname must not be empty
        if len(responses["HOSTNAME"]) == 0:
            errors.append("Hostname must not be empty.")

        #hostname needs to have valid chars
        if re.search('[^a-z0-9-]', responses["HOSTNAME"]):
            errors.append(
                "Hostname must contain only alphanumeric and hyphen.")

        #domain must be under 180 chars
        if len(responses["DNS_DOMAIN"]) >= 180:
            errors.append("Domain must be under 180 chars.")

        #domain must not be empty
        if len(responses["DNS_DOMAIN"]) == 0:
            errors.append("Domain must not be empty.")

        #domain needs to have valid chars
        if re.match('[^a-z0-9-.]', responses["DNS_DOMAIN"]):
            errors.append(
                "Domain must contain only alphanumeric, period and hyphen.")
        #ensure external DNS is valid
        if len(responses["DNS_UPSTREAM"]) == 0:
            #We will allow empty if user doesn't need external networking
            #and present a strongly worded warning
            msg = "If you continue without DNS, you may not be able to access"\
                  + " external data necessary for installation needed for " \
                  + "some OpenStack Releases."

            dialog.display_dialog(
                self, widget.TextLabel(msg), "Empty DNS Warning")

        else:
            #external DNS must contain only numbers, periods, and commas
            #Needs more serious ip address checking
            if re.match('[^0-9.,]', responses["DNS_UPSTREAM"]):
                errors.append(
                    "External DNS must contain only IP addresses and commas.")
            #ensure test DNS name isn't empty
            if len(responses["TEST_DNS"]) == 0:
                errors.append("Test DNS must not be empty.")
            #Validate first IP address
            try:
                if netaddr.valid_ipv4(responses["DNS_UPSTREAM"].split(",")[0]):
                    DNS_UPSTREAM = responses["DNS_UPSTREAM"].split(",")[0]
                else:
                    errors.append("Not a valid IP address for External DNS: %s"
                                  % responses["DNS_UPSTREAM"])

                #Try to resolve with first address
                if not self.checkDNS(DNS_UPSTREAM):
                    #Warn user that DNS resolution failed, but continue
                    msg = "Unable to resolve %s.\n\n" % responses['TEST_DNS']\
                          + "Possible causes for DNS failure include:\n"\
                          + "* Invalid DNS server\n"\
                          + "* Invalid gateway\n"\
                          + "* Other networking issue\n\n"\
                          + "Fuel Setup can save this configuration, but "\
                          + "you may want to correct your settings."
                    dialog.display_dialog(self, widget.TextLabel(msg),
                                          "DNS Failure Warning")
                    self.parent.refreshScreen()
            except Exception:
                errors.append("Not a valid IP address for External DNS: %s"
                              % responses["DNS_UPSTREAM"])

        if len(errors) > 0:
            self.parent.footer.set_text("Error: %s" % (errors[0]))
            log.error("Errors: %s %s" % (len(errors), errors))
            return False
        else:
            self.parent.footer.set_text("No errors found.")
            return responses

    def apply(self, args):
        responses = self.check(args)
        if responses is False:
            log.error("Check failed. Not applying")
            log.error("%s" % (responses))
            return False

        self.save(responses)
        #Update network details so we write correct IP address
        self.getNetwork()
        #Apply hostname
        expr = 'HOSTNAME=.*'
        replace.replaceInFile("/etc/sysconfig/network", expr,
                              "HOSTNAME=%s.%s"
                              % (responses["HOSTNAME"],
                                 responses["DNS_DOMAIN"]))
        #remove old hostname from /etc/hosts
        f = open("/etc/hosts", "r")
        lines = f.readlines()
        f.close()
        with open("/etc/hosts", "w") as etchosts:
            for line in lines:
                if "localhost" in line:
                    etchosts.write(line)
                elif responses["HOSTNAME"] in line \
                        or self.oldsettings["HOSTNAME"] \
                        or self.netsettings[self.parent.managediface]['addr'] \
                        in line:
                    continue
                else:
                    etchosts.write(line)
            etchosts.close()

        #append hostname and ip address to /etc/hosts
        with open("/etc/hosts", "a") as etchosts:
            if self.netsettings[self.parent.managediface]["addr"] != "":
                managediface_ip = self.netsettings[
                    self.parent.managediface]["addr"]
            else:
                managediface_ip = "127.0.0.1"
            etchosts.write(
                "%s   %s.%s %s\n" % (managediface_ip, responses["HOSTNAME"],
                                     responses['DNS_DOMAIN'],
                                     responses["HOSTNAME"]))
            etchosts.close()

        def make_resolv_conf(filename):
            with open(filename, 'w') as f:
                f.write("search %s\n" % responses['DNS_SEARCH'])
                f.write("domain %s\n" % responses['DNS_DOMAIN'])
                for upstream_dns in responses['DNS_UPSTREAM'].split(','):
                    f.write("nameserver %s\n" % upstream_dns)

        # Create a temporary resolv.conf so DNS works before the cobbler
        # container is up and running.
        # TODO(asheplyakov): puppet does a similar thing, perhaps we can
        # use the corresponding template instead of duplicating it here.
        make_resolv_conf('/etc/resolv.conf')

        return True

    def cancel(self, button):
        ModuleHelper.cancel(self, button)

    def load(self):
        #Read in yaml
        defaultsettings = Settings().read(self.parent.defaultsettingsfile)
        oldsettings = defaultsettings
        oldsettings.update(Settings().read(self.parent.settingsfile))

        oldsettings = Settings().read(self.parent.settingsfile)
        for setting in self.defaults.keys():
            try:
                if "/" in setting:
                    part1, part2 = setting.split("/")
                    self.defaults[setting]["value"] = oldsettings[part1][part2]
                else:
                    self.defaults[setting]["value"] = oldsettings[setting]
            except Exception:
                log.warning("No setting named %s found." % setting)
                continue
        #Read hostname if it's already set
        try:
            hostname, sep, domain = os.uname()[1].partition('.')
            oldsettings["HOSTNAME"] = hostname
            oldsettings["DNS_DOMAIN"] = domain
            oldsettings["DNS_SEARCH"] = domain
        except Exception:
            log.warning("Unable to look up system hostname")
        return oldsettings

    def save(self, responses):
        ## Generic settings start ##
        newsettings = dict()
        for setting in responses.keys():
            if "/" in setting:
                part1, part2 = setting.split("/")
                if part1 not in newsettings:
                #We may not touch all settings, so copy oldsettings first
                    newsettings[part1] = self.oldsettings[part1]
                newsettings[part1][part2] = responses[setting]
            else:
                newsettings[setting] = responses[setting]
        ## Generic settings end ##

        #log.debug(str(newsettings))
        Settings().write(newsettings,
                         defaultsfile=self.parent.defaultsettingsfile,
                         outfn=self.parent.settingsfile)

        #Set oldsettings to reflect new settings
        self.oldsettings = newsettings
        #Update self.defaults
        for index, fieldname in enumerate(self.fields):
            if fieldname != "blank":
                self.defaults[fieldname]['value'] = newsettings[fieldname]

    def checkDNS(self, server):
        #Note: Python's internal resolver caches negative answers.
        #Therefore, we should call dig externally to be sure.

        noout = open('/dev/null', 'w')
        dns_works = subprocess.call(["dig", "+short", "+time=3",
                                     "+retries=1",
                                     self.defaults["TEST_DNS"]['value'],
                                     "@%s" % server], stdout=noout,
                                    stderr=noout)
        if dns_works != 0:
            return False
        else:
            return True

    def getNetwork(self):
        ModuleHelper.getNetwork(self)

    def getDHCP(self, iface):
        return ModuleHelper.getDHCP(iface)

    def get_default_gateway_linux(self):
        return ModuleHelper.get_default_gateway_linux()

    def radioSelect(self, current, state, user_data=None):
        pass

    def refresh(self):
        pass

    def screenUI(self):
        return ModuleHelper.screenUI(self, self.header_content, self.fields,
                                     self.defaults)


# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# ipmisim - Fake IPMI simulator for testing, forked from Conpot
# Maintainer - Rohit Yadav <bhaisaab@apache.org>
# Original Author: Peter Sooky <xsooky00@stud.fit.vubtr.cz>
# Brno University of Technology, Faculty of Information Technology

        # | FNC:CMD   | NetFunc         | Command                             |
        # | --------- | ----------------|------------------------------------ |
        # | 0x00:0x00 | Chassis         | Chassis Capabilities                |
        # | 0x00:0x01 | Chassis         | Get Chassis Status                  |
        # | 0x00:0x02 | Chassis         | Chassis Control                     |
        # | 0x00:0x08 | Chassis         | Set System Boot Options             |
        # | 0x00:0x09 | Chassis         | Get System Boot Options             |
        # | 0x04:0x2D | Sensor/Event    | Get Sensor Reading                  |
        # | 0x04:0x2F | Sensor/Event    | Get Sensor Type                     |
        # | 0x04:0x30 | Sensor/Event    | Set Sensor Reading and Event Status |
        # | 0x06:0x01 | App             | Get Device ID                       |
        # | 0x06:0x02 | App             | Cold Reset                          |
        # | 0x06:0x03 | App             | Warm Reset                          |
        # | 0x06:0x04 | App             | Get Self Test Results               |
        # | 0x06:0x08 | App             | Get Device GUID                     |
        # | 0x06:0x22 | App             | Reset Watchdog Timer                |
        # | 0x06:0x24 | App             | Set Watchdog Timer                  |
        # | 0x06:0x2E | App             | Set BMC Global Enables              |
        # | 0x06:0x31 | App             | Get Message Flags                   |
        # | 0x06:0x35 | App             | Read Event Message Buffer           |
        # | 0x06:0x36 | App             | Get BT Interface Capabilities       |
        # | 0x06:0x40 | App             | Set Channel Access                  |
        # | 0x06:0x41 | App             | Get Channel Access                  |
        # | 0x06:0x42 | App             | Get Channel Info Command            |
        # | 0x0A:0x10 | Storage         | Get FRU Inventory Area Info         |
        # | 0x0A:0x11 | Storage         | Read FRU Data                       |
        # | 0x0A:0x12 | Storage         | Write FRU Data                      |
        # | 0x0A:0x40 | Storage         | Get SEL Info                        |
        # | 0x0A:0x42 | Storage         | Reserve SEL                         |
        # | 0x0A:0x44 | Storage         | Add SEL Entry                       |
        # | 0x0A:0x48 | Storage         | Get SEL Time                        |
        # | 0x0A:0x49 | Storage         | Set SEL Time                        |
        # | 0x0C:0x01 | Transport       | Set LAN Configuration Parameters    |
        # | 0x0C:0x02 | Transport       | Get LAN Configuration Parameters    |
        # | 0x2C:0x00 | Group Extension | Group Extension Command             |
        # | 0x2C:0x03 | Group Extension | Get Power Limit                     |
        # | 0x2C:0x04 | Group Extension | Set Power Limit                     |
        # | 0x2C:0x05 | Group Extension | Activate/Deactivate Power Limit     |
        # | 0x2C:0x06 | Group Extension | Get Asset Tag                       |
        # | 0x2C:0x08 | Group Extension | Set Asset Tag                       |

import logging
import os
import sys
from pyghmi.ipmi.bmc import Bmc

import packet

logger = logging.getLogger('fakebmc')
logger.setLevel(logging.DEBUG)
ch = logging.StreamHandler()
ch.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(asctime)s : %(levelname)s : %(name)s : %(message)s')
ch.setFormatter(formatter)
logger.addHandler(ch)

METAL_AUTH_TOKEN = os.getenv("METAL_AUTH_TOKEN")
METAL_SERVER_UUID = os.getenv("METAL_SERVER_UUID")
METAL_SERVER_IPXE_URL = os.getenv("METAL_SERVER_IPXE_URL")
METAL_SERVER_IPMI_IP = os.getenv("METAL_SERVER_IPMI_IP")
METAL_SERVER_IPMI_GW_IP = os.getenv("METAL_SERVER_IPMI_GW_IP")
METAL_SERVER_IPMI_MAC = os.getenv("METAL_SERVER_IPMI_MAC")

if METAL_AUTH_TOKEN is None:
    logger.error("OS ENV variable METAL_AUTH_TOKEN= must be set")
    sys.exit(1)
elif METAL_SERVER_UUID is None:
    logger.error("OS ENV variable METAL_SERVER_UUID= must be set")
    sys.exit(1)
elif METAL_SERVER_IPXE_URL is None:
    logger.error("OS ENV variable METAL_SERVER_IPXE_URL= must be set")
    sys.exit(1)
elif METAL_SERVER_IPMI_IP is None:
    logger.error("OS ENV variable METAL_SERVER_IPMI_IP= must be set")
    sys.exit(1)
elif METAL_SERVER_IPMI_GW_IP is None:
    logger.error("OS ENV variable METAL_SERVER_IPMI_GW_IP= must be set")
    sys.exit(1)
elif METAL_SERVER_IPMI_MAC is None:
    logger.error("OS ENV variable METAL_SERVER_IPMI_MAC= must be set")
    sys.exit(1)

manager = packet.Manager(auth_token=METAL_AUTH_TOKEN)
virtualmedia = 'mounted'

class FakeBmc(Bmc):

    def __init__(self, authdata):
        self.authdata = authdata
        # Initialize fake BMC config
        self.deviceid = 0x24
        self.revision = 0x10
        self.firmwaremajor = 0x10
        self.firmwareminor = 0x1
        self.ipmiversion = 2
        self.additionaldevices = 0
        self.mfgid = 0xf
        self.prodid = 0xe

        self.powerstate = 'off'
        self.bootdevice = 'default'
        logger.info('IPMI BMC initialized.')

    def get_boot_device(self):
        logger.info('IPMI BMC Get_Boot_Device request.')
        return self.bootdevice

    def set_boot_device(self, bootdevice):
        server = manager.get_device(METAL_SERVER_UUID)
        logger.info('IPMI BMC Set_Boot_Device request.')
        logger.info('IPMI BMC bootdevice request is for %s' % bootdevice)
        logger.info('Metal Server PXE state is %s' % server.always_pxe)
        if bootdevice == 'network':
            logger.info('Metal Server iPXE URL being set to: %s' % METAL_SERVER_IPXE_URL)
            logger.info('Metal Server always_ipxe being set to True')
            server.ipxe_script_url = METAL_SERVER_IPXE_URL
            server.always_pxe = True
            server.update()
        if bootdevice == 'hd':
            logger.info('Metal Server always_ipxe being set to False')
            server.always_pxe = False
            server.update()
        self.bootdevice = bootdevice

    def cold_reset(self):
        server = manager.get_device(METAL_SERVER_UUID)
        logger.info('IPMI BMC Cold_Reset request.')
        server.reboot()
        self.powerstate = 'off'
        #self.bootdevice = 'default'

    def get_power_state(self):
        server = manager.get_device(METAL_SERVER_UUID)
        logger.info('IPMI BMC Get_Power_State request.')
        logger.info('Metal Server State: %s' % server.state)
        if server.state == 'active':
            self.powerstate = 'on'
        if server.state == 'powering_on':
            logger.debug('Metal Server is powering on')
            self.powerstate = 'on'
        if server.state == 'powering_off':
            logger.debug('Metal Server is powering off')
            self.poweroff = 'off'
        elif server.state == 'inactive':
            self.powerstate = 'off'
        return self.powerstate

    def power_off(self):
        server = manager.get_device(METAL_SERVER_UUID)
        logger.info('IPMI BMC Power_Off request.')
        logger.info('Metal Server State: %s' % server.state)
        # IPMI will ask to turn a server off but expect a return of "on" till it is "off"
        # Chassis Power Control: Up/Off
        # IPMItool will keep the session open after the first request and request again till Down/Off
        if server.state == 'active':
            try:
                logger.info('Metal API: power OFF %s', METAL_SERVER_UUID)
                server.power_off()
                self.powerstate = 'on'
            except packet.baseapi.ResponseError as e:
                logger.error('Metal API returned %s as error to power OFF request' % e)
                self.powerstate = 'unknown'
        elif server.state == 'inactive':
            self.powerstate = 'off'
            logger.debug('Metal server is already off')
        else:
            logger.debug('Metal server will return state unknown, expected')
            self.powerstate = 'unknown'

    def power_on(self):
        server = manager.get_device(METAL_SERVER_UUID)
        logger.info('IPMI BMC Power_On request.')
        logger.info('Metal Server State: %s' % server.state)
        # IPMI will ask to turn a server on but expect a return of "off" till it is "on"
        # Chassis Power Control: Down/On
        if server.state == 'active':
            self.powerstate = 'on'
            logger.debug('Metal server is already on')
        elif server.state == 'inactive':
            try:
                logger.info('Metal API: power ON %s', METAL_SERVER_UUID)
                server.power_on()
            except packet.baseapi.ResponseError as e:
                logger.error('Metal API returned %s as error to power ON request' % e)
                self.powerstate = 'unknown'
        else:
            self.powerstate = 'unknown'
        self.powerstate

    def power_reset(self):
        logger.info('IPMI BMC Power_Reset request.')
        server = manager.get_device(METAL_SERVER_UUID)
        if server.state == 'inactive':
            server.power_on()
            try:
                logger.debug('Metal Server: is OFF for Power_Reset call, issuing power ON')
                self.powerstate = 'on'
            except packet.baseapi.ResponseError as e:
                logger.error('Metal API returned %s as error to power ON request' % e)
                self.powerstate = 'unknown'
        else:
            logger.debug('Metal API: REBOOT server')
            server.reboot()
        # warm boot
        self.powerstate = 'on'

    def power_cycle(self):
        logger.info('IPMI BMC Power_Cycle request.')
        server = manager.get_device(METAL_SERVER_UUID)
        if server.state == 'inactive':
            server.power_on()
            try:
                logger.debug('Metal Server: is OFF for Power_Cycle call, issuing power ON')
                self.powerstate = 'on'
            except packet.baseapi.ResponseError as e:
                logger.error('Metal API returned %s as error to power ON request' % e)
                self.powerstate = 'unknown'
        else:
            logger.debug('Metal API: REBOOT server')
            server.reboot()
        # cold boot
        self.powerstate = 'off'
        self.powerstate = 'on'

    def power_shutdown(self):
        logger.info('IPMI BMC Power_Shutdown request.')
        self.powerstate = 'off'

    # This is a horrible horrible thing to do
    # Todo redo all of this
    # https://opendev.org/x/pyghmi/src/branch/master/pyghmi/ipmi/bmc.py#L162

    def custom_handle_raw_request(self, request, session):

        global virtualmedia
        logger.debug('CUSTOM HANDLER command is %s', str.format('0x{:02X}', int(str(request["command"]), 16)))
        #try:
        if request['netfn'] == 6:
            # Channel info
            if request["command"] == 0x42: # 0x06 0x42
                self.session._send_ipmi_net_payload(data=get_channel_info())
            elif request["command"] == 0x41: # 0x06 0x41 (automatic subcall of 0x42)
                self.session._send_ipmi_net_payload(data=get_channel_settings())
        elif request['netfn'] == 10:
            # fru
            if request["command"] == 0x10: # 0x0a 0x10 (automatic subcall of 0x42)
                self.session._send_ipmi_net_payload(data=get_fru_inventory_area_info())
            elif request["command"] == 0x11: # 0x0a 0x11 (automatic subcall of 0x42)
                # all of this is fru print 0, fru list currently still broken
                zeros_number = len([num for num in request.get("data") if num == 0x00])
                twenty_number = len([num for num in request.get("data") if num == 0x20])
                if zeros_number == 3 and 0x08 in request.get("data"):
                    logger.info('IPMI fru print 0 called likely')
                    self.session._send_ipmi_net_payload(data=read_fru_data1())
                elif zeros_number == 2 and 0x08 in request.get("data") and 0x02 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=read_fru_data2())
                elif zeros_number == 2 and 0x08 in request.get("data") and 0x18 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=read_fru_data3())
                elif zeros_number == 2 and 0x20 in request.get("data") and 0x02 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=read_fru_data4())
                elif zeros_number == 2 and twenty_number == 2:
                    self.session._send_ipmi_net_payload(data=read_fru_data5())
                elif zeros_number == 2 and 0x40 in request.get("data") and 0x20 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=read_fru_data6())
                elif zeros_number == 2 and 0x60 in request.get("data") and 0x18 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=read_fru_data7())
                elif zeros_number == 2 and 0x78 in request.get("data") and 0x02 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=read_fru_data8())
                elif zeros_number == 2 and 0x78 in request.get("data") and 0x20 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=read_fru_data9())
                elif zeros_number == 2 and 0x98 in request.get("data") and 0x20 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=read_fru_data10())
                elif zeros_number == 2 and 0xb8 in request.get("data") and 0x10 in request.get("data"): # begin air(private note)
                    self.session._send_ipmi_net_payload(data=read_fru_data11())
        elif request['netfn'] == 12:
            if request["command"] == 0x02: # 0x0c 0x02
                zeros_number = len([num for num in request.get("data") if num == 0x00])
                ones_number = len([num for num in request.get("data") if num == 0x01])
                if zeros_number == 3 and 0x01 in request.get("data"):
                    logger.info('IPMI lan print called likely')
                    self.session._send_ipmi_net_payload(data=get_lan_1())
                elif zeros_number == 2 and ones_number == 2:
                    self.session._send_ipmi_net_payload(data=get_lan_2())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x02 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_3())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x04 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_4())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x03 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_5()) #this function is ipmi lan IP
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x06 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_6())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x05 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_7()) # ipmi mac
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x10 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_8())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x07 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_9())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x0a in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_10())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x0b in request.get("data"):
                    self.session._send_ipmi_net_payload(code=0x80)  # get_lan_11 shortcut
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x0c in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_12()) # func is ipmi GW ip
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x0d in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_13())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x0e in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_14())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x0f in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_15())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x14 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_16())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x15 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_17())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x16 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_18())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x17 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_19())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x18 in request.get("data"):
                    self.session._send_ipmi_net_payload(data=get_lan_20())
                elif zeros_number == 2 and 0x01 in request.get("data") and 0x1a in request.get("data"):
                    logger.debug('ipmi client request lan print should be complete')
                    self.session._send_ipmi_net_payload(code=0x80)  # get_lan_21 shortcut
        elif request['netfn'] == 60: # 0x3c, special / secret virtualmedia ipmi path for supermicro based lifecycle controllers. This is all undocumented
            if request['command'] == 0x03: # This is the get virtualmedia mount status command '0x3c 0x03'
                logger.info("IPMI request for virtualmedia nfs status received, status: %s", virtualmedia)
                if virtualmedia == 'mounted':
                    # Hex for ' 00\n'
                    self.session._send_ipmi_net_payload(data=[0x00])
                elif virtualmedia == 'dismounted':
                    self.session._send_ipmi_net_payload(code=0x00)
                else:
                    self.session._send_ipmi_net_payload(code=0x00)
            elif request['command'] == 0x01:
                # This should come after but this is stupid python class / attribute stuff
                self.session._send_ipmi_net_payload(code=0x00)
                server = manager.get_device(METAL_SERVER_UUID)
                # This is the virtual media set NFS image command for SuperMicro, data payloads are around that. Impossible to predict incoming data payload as it will be dynamic as in:
                # 0x3c 0x01 0x02 %s 0x00' %(self.hex_convert(image_filename)
                server.ipxe_script_url = METAL_SERVER_IPXE_URL
                server.always_pxe = True
                server.update()
                virtualmedia = 'mounted'
                logger.info("IPMI request for virtualmedia nfs set received, iPXE enabled")
            elif request['command'] == 0x00:
                self.session._send_ipmi_net_payload(code=0x00)
                server = manager.get_device(METAL_SERVER_UUID)
                server.always_pxe = False
                server.update()
                virtualmedia = 'dismounted'
                logger.info("IPMI request for virtualmedia nfs UNSET received, iPXE disabled")
            elif request['command'] == 0x02: # start virtualmedia against configured NFS mount
                self.session._send_ipmi_net_payload(code=0x00)
                logger.info("IPMI request for virtualmedia nfs start received")

#Taken from
# https://github.com/kurokobo/virtualbmc-for-vsphere/blob/master/vbmc4vsphere/vbmc.py#L350
def get_channel_info():
    logger.debug('get_channel_info requested')
    channel_data = [
        0x02,  # channel number = 2
        0x04,  # channel medium type = 802.3 LAN
        0x01,  # channel protocol type = IPMB-1.0
        0x80,  # session support = multi-session
        0xF2,  # vendor id = 7154
        0x1B,  # vendor id = 7154
        0x00,  # vendor id = 7154
        0x00,  # reserved
        0x00,  # reserved
    ]
    return channel_data

#Taken from
# https://github.com/kurokobo/virtualbmc-for-vsphere/blob/master/vbmc4vsphere/vbmc.py#L350
def get_channel_settings():
    logger.debug('get_channel_info_settings requested')
    channel_settings = [
        0x12,
        0x04,
    ]
    return channel_settings

def get_fru_inventory_area_info():
    logger.debug('get_fru_inventory_area_info requested')
    fru_area_info = [
        0x00,
        0x01,
        0x00,

    ]
    return fru_area_info

def read_fru_data1():
    logger.debug('read_fru_data1')
    fru_data1 = [
        0x08,
        0x01,
        0x00,
        0x01,
        0x04,
        0x0f,
        0x00,
        0x00,
        0xeb,
    ]
    return fru_data1

def read_fru_data2():
    fru_data2 = [
        0x02,
        0x01,
        0x03,
    ]
    return fru_data2

def read_fru_data3():
    fru_data3 = [
        0x18,
        0x01,
        0x03,
        0x17,
        0x00,
        0xcd,
        0x51,
        0x54,
        0x46,
        0x43,
        0x4f,
        0x43,
        0x37,
        0x31,
        0x35,
        0x30,
        0x32,
        0x31,
        0x45,
        0xc1,
        0x00,
        0x00,
        0x00,
        0x00,
        0x22,
    ]
    return fru_data3

def read_fru_data4():
    fru_data4 = [
        0x02,
        0x01,
        0x0b,
    ]
    return fru_data4

def read_fru_data5():
    fru_data5 = [
        0x20,
        0x01,
        0x0b,
        0x19,
        0x79,
        0xe9,
        0xaa,
        0xd3,
        0x51,
        0x75,
        0x61,
        0x6e,
        0x74,
        0x61,
        0x20,
        0x43,
        0x6f,
        0x6d,
        0x70,
        0x75,
        0x74,
        0x65,
        0x72,
        0x20,
        0x49,
        0x6e,
        0x63,
        0xd5,
        0x53,
        0x32,
        0x42,
        0x50,
        0x2d,
    ]
    return fru_data5

def read_fru_data6():
    fru_data6 = [
        0x20,
        0x4d,
        0x42,
        0x20,
        0x28,
        0x64,
        0x75,
        0x61,
        0x6c,
        0x20,
        0x31,
        0x47,
        0x20,
        0x4c,
        0x6f,
        0x4d,
        0x29,
        0xce,
        0x51,
        0x54,
        0x46,
        0x35,
        0x4f,
        0x43,
        0x37,
        0x31,
        0x34,
        0x30,
        0x30,
        0x33,
        0x31,
        0x34,
        0xcb,
    ]
    return fru_data6

def read_fru_data7():
    fru_data7 = [
        0x18,
        0x33,
        0x31,
        0x53,
        0x32,
        0x42,
        0x4d,
        0x42,
        0x30,
        0x30,
        0x35,
        0x30,
        0xc3,
        0x31,
        0x2e,
        0x34,
        0xc0,
        0xc0,
        0xc2,
        0x38,
        0x30,
        0xc1,
        0x00,
        0x00,
        0x4b,
    ]
    return fru_data7

def read_fru_data8():
    fru_data8 = [
        0x02,
        0x01,
        0x0a,
    ]
    return fru_data8

def read_fru_data9():
    fru_data9 = [
        0x20,
        0x01,
        0x0a,
        0x19,
        0xd3,
        0x51,
        0x75,
        0x61,
        0x6e,
        0x74,
        0x61,
        0x20,
        0x43,
        0x6f,
        0x6d,
        0x70,
        0x75,
        0x74,
        0x65,
        0x72,
        0x20,
        0x49,
        0x6e,
        0x63,
        0xd6,
        0x44,
        0x35,
        0x31,
        0x42,
        0x50,
        0x2d,
        0x31,
        0x55,
    ]
    return fru_data9

def read_fru_data10():
    fru_data10 = [
        0x20,
        0x20,
        0x28,
        0x64,
        0x75,
        0x61,
        0x6c,
        0x20,
        0x31,
        0x47,
        0x20,
        0x4c,
        0x6f,
        0x4d,
        0x29,
        0xcb,
        0x32,
        0x30,
        0x53,
        0x32,
        0x42,
        0x42,
        0x55,
        0x30,
        0x32,
        0x4b,
        0x30,
        0x00,
        0xcd,
        0x51,
        0x54,
        0x46,
        0x43,
    ]
    return fru_data10

def read_fru_data11():
    logger.debug('read_fru_data11 reached, `fru print 0` should be complete')
    fru_data11 = [
        0x10,
        0x4f,
        0x43,
        0x37,
        0x31,
        0x35,
        0x30,
        0x32,
        0x31,
        0x45,
        0x00,
        0xc3,
        0x31,
        0x2e,
        0x34,
        0xc1,
        0xd9,
    ]
    return fru_data11

def get_lan_1():
    lan_data1 = [
        0x11,
        0x00,
    ]
    return lan_data1

def get_lan_2():
    lan_data2 = [
        0x11,
        0x17,
    ]
    return lan_data2

def get_lan_3():
    lan_data3 = [
        0x11,
        0x16,
        0x16,
        0x16,
        0x16,
        0x16,
    ]
    return lan_data3

def get_lan_4():
    lan_data4 = [
        0x11,
        0x02,
    ]
    return lan_data4

def get_lan_5():
    ipmi_ip = METAL_SERVER_IPMI_IP

    ip_response_data = [
        0x11,
    ]
    # https://stackoverflow.com/a/41225217
    for portion in ipmi_ip.split('.'):
        portion_to_hex = hex(int(portion)+256)[3:]
        ip_response_data.append(int(portion_to_hex, 16))
    return ip_response_data

def get_lan_6():
    lan_data6 = [
        0x11,
        0xff,
        0xff,
        0xff,
        0x00,
    ]
    return lan_data6

def get_lan_7():
    ipmi_mac = METAL_SERVER_IPMI_MAC

    mac_response_data = [
        0x11,
    ]
    #str.format('0x{:02X}'
    for portion in ipmi_mac.split(':'):
        # nothing is really being hexed here, str format to int
        portion_to_hex = str.format('0x{0}', portion)
        mac_response_data.append(int(portion_to_hex, 16))
    return mac_response_data

def get_lan_8():
    lan_data8 = [
        0x11,
        0x70,
        0x75,
        0x62,
        0x6c,
        0x69,
        0x63,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
    ]
    return lan_data8

def get_lan_9():
    lan_data9 = [
        0x11,
        0x00,
        0x00,
        0x00,
        0x00,
    ]
    return lan_data9

def get_lan_10():
    lan_data10 = [
        0x11,
        0x02,
    ]
    return lan_data10

def get_lan_12():
    ipmi_gw_ip = METAL_SERVER_IPMI_GW_IP

    ip_response_data = [
        0x11,
    ]
    # https://stackoverflow.com/a/41225217
    for portion in ipmi_gw_ip.split('.'):
        portion_to_hex = hex(int(portion)+256)[3:]
        ip_response_data.append(int(portion_to_hex, 16))
    return ip_response_data

    lan_data12 = [
        0x11,
        0x0a,
        0xfa,
        0x1f,
        0x01,
    ]
    return lan_data12

def get_lan_13():
    lan_data13 = [
        0x11,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
    ]
    return lan_data13

def get_lan_14():
    lan_data14 = [
        0x11,
        0x00,
        0x00,
        0x00,
        0x00,
    ]
    return lan_data14

def get_lan_15():
    lan_data15 = [
        0x11,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
    ]
    return lan_data15

def get_lan_16():
    lan_data16 = [
        0x11,
        0x00,
        0x00,
    ]
    return lan_data16

def get_lan_17():
    lan_data17 = [
        0x11,
        0x00,
    ]
    return lan_data17

def get_lan_18():
    lan_data18 = [
        0x11,
        0x08,
    ]
    return lan_data18

def get_lan_19():
    lan_data19 = [
        0x11,
        0x00,
        0x01,
        0x02,
        0x03,
        0x06,
        0x07,
        0x08,
        0x0b,
        0x0c,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
        0x00,
    ]
    return lan_data19

def get_lan_20():
    lan_data20 = [
        0x11,
        0x00,
        0x40,
        0x44,
        0x00,
        0x44,
        0x04,
        0x40,
        0x04,
        0x00,
    ]
    return lan_data20


import torch
import numpy as np
import json

class Predictor(object):
    def __init__(self, model, vectorizer, use_cuda=False):
        """
        Predictor class to evaluate for a given model.
        Args:
            model (seq2seq.models): trained model. This can be loaded from a checkpoint
                using `seq2seq.util.checkpoint.load`
            src_vocab (seq2seq.dataset.vocabulary.Vocabulary): source sequence vocabulary
            tgt_vocab (seq2seq.dataset.vocabulary.Vocabulary): target sequence vocabulary
        """
        self.use_cuda = use_cuda
        if use_cuda:
            self.model = model.cuda()
        else:
            self.model = model.cpu()
        self.model.eval()
        self.vectorizer = vectorizer

    def predict(self, src_seq, num_exams, topics=None, max_length=None, use_structure=False):
        """ Make prediction given `src_seq` as input.

        Args:
            topics (list): list of topics (max 2) for the title. Use for contextual generation
            src_seq (list): list of tokens in source language

        Returns:
            tgt_seq (list): list of tokens in target language as predicted
            by the pre-trained model
        """

        torch.set_grad_enabled(False)
        text = []
        for tok in src_seq:
            if tok in self.vectorizer.word2idx:
                text.append(self.vectorizer.word2idx[tok])
            else:
                text.append(3)

        if topics:
            topics = torch.LongTensor(topics).view(1,-1)
            if self.use_cuda:
                topics = topics.cuda()
        else:
            topics = None

        # If provided by the user, use that, else let it be the default max_length from training data.
        if max_length:
            self.model.decoder.max_length = max_length

        input_variable = torch.LongTensor(text).view(1, -1)
        if self.use_cuda:
            input_variable = input_variable.cuda()

        input_lengths = torch.LongTensor([len(src_seq)])

        prev_generated_seq = None
        structure_abstracts = None
        outputs = []
        for i in range(num_exams):
            _, _, other = \
                self.model(input_variable, prev_generated_seq, input_lengths, topics=topics, structure_abstracts=structure_abstracts)
            length = other['length'][0]

            tgt_id_seq = [other['sequence'][di][0].item() for di in range(length)]
            tgt_seq = [self.vectorizer.idx2word[tok] for tok in tgt_id_seq]
            output = ' '.join([i for i in tgt_seq if i != '<PAD>' and i != '<EOS>' and i != '<SOS>'])
            outputs.append(output)
            prev_generated_seq = torch.LongTensor(tgt_id_seq).view(1, -1).cuda() if self.use_cuda else torch.LongTensor(tgt_id_seq).view(1, -1)
            if use_structure:
                structure_abstracts = [other['gen_labels'][di] for di in range(length)]
                structure_abstracts = torch.LongTensor(structure_abstracts).view(1, -1).cuda() if self.use_cuda else torch.LongTensor(structure_abstracts).view(1, -1)
        return outputs

    def predict_batch(self, source, input_lengths, num_exams):
        torch.set_grad_enabled(False)
        output_seq = []
        input_variables = source
        for i in range(source.size(0)):
            title_id_seq = [input_variables[i][di].item() for di in range(input_lengths[i])]
            title_seq = [self.vectorizer.idx2word[tok] for tok in title_id_seq]
            title = ' '.join([k for k in title_seq if k != '<PAD>' and k != '<EOS>' and k != '<SOS>'])
            output_seq.append([title])
        prev_generated_seq = None
        for k in range(num_exams):
            _, _, other = \
                self.model(input_variables, prev_generated_seq, input_lengths)
            length = other['length']
            sequence = torch.stack(other['sequence'], 1).squeeze(2)
            prev_generated_seq = self._mask(sequence)
            for i in range(len(length)):
                opt_id_seq = [other['sequence'][di][i].item() for di in range(length[i])]
                opt_seq = [self.vectorizer.idx2word[tok] for tok in opt_id_seq]
                output = ' '.join([k for k in opt_seq if k != '<PAD>' and k != '<EOS>' and k != '<SOS>'])
                output_seq[i].append(output)
        return output_seq

    # Mask variable
    def _mask(self, prev_generated_seq):
        prev_mask = torch.eq(prev_generated_seq, 1).cpu().data.numpy()
        lengths = np.argmax(prev_mask,axis=1)
        max_len = prev_generated_seq.size(1)
        mask = []
        for i in range(prev_generated_seq.size(0)):
            if lengths[i] == 0:
                mask_line = [0] * max_len
            else:
                mask_line = [0] * lengths[i]
                mask_line.extend([1] * (max_len - lengths[i]))
            mask.append(mask_line)
        mask = torch.ByteTensor(mask)
        if self.use_cuda:
            mask = mask.cuda()
        return prev_generated_seq.data.masked_fill_(mask, 0)

    def preeval_batch(self, test_loader, abs_len, num_exams, use_topics=False, use_labels=False):
        torch.set_grad_enabled(False)
        org = {}
        refs = {}
        cands = []
        for i in range(num_exams):
            cands.append({})
        i = 0
        for batch_idx, data in enumerate(test_loader):
            topics = data[3] if use_topics else None
            # Since this is test time, we won't feed any structure labels from the test set.
            # The model has to figure them out for all the abstracts.
            structure_abstracts = None

            input_variables = data[0]
            target_variables = data[1]
            input_lengths = data[2]

            for j in range(input_variables.size(0)):
                i += 1
                ref = self.prepare_for_bleu(target_variables[j])
                refs[i] = [ref]
                org[i] = data[-1][j]
            prev_generated_seq = None
            for k in range(num_exams):
                _, _, other = \
                    self.model(input_variables, prev_generated_seq, input_lengths, topics=topics, structure_abstracts=structure_abstracts)
                length = other['length']
                sequence = torch.stack(other['sequence'], 1).squeeze(2)
                prev_generated_seq = self._mask(sequence)
                for j in range(len(length)):
                    out_seq = [other['sequence'][di][j] for di in range(length[j])]
                    out = self.prepare_for_bleu(out_seq)
                    cands[k][len(cands[k]) + 1] = out
                if use_labels:
                    structure_abstracts = torch.stack(other['gen_labels'], 1).squeeze(2)
            if i % 100 == 0:
                print("Percentages:  %.4f" % (i/float(abs_len)))
        return cands, refs, org

    def prepare_for_bleu(self, sentence):
        sent=[x.item() for x in sentence if x.item() != 0 and x.item() != 1 and x.item() != 2]
        sent = ' '.join([str(x) for x in sent])
        return sent

    def evaluate_abstracts(self, filename, vectorizer):
        refs = {}
        cand = {}
        with open(filename) as f:
            for i, line in enumerate(f):
                j = json.loads(line.strip())
                original_abstract = vectorizer.source_to_tokens(j["original"])
                generated_abstract = vectorizer.source_to_tokens(j["generated"])
                refs[i] = [self.prepare_for_bleu(original_abstract)]
                cand[i] = self.prepare_for_bleu(generated_abstract)
        return cand, refs

    def predict_seq_title(self, title, sec_seq, num_exams):
        """ Make prediction given `src_seq` as input.

        Args:
            src_seq (list): list of tokens in source language

        Returns:
            tgt_seq (list): list of tokens in target language as predicted
            by the pre-trained model
        """
        torch.set_grad_enabled(False)
        text = []
        for tok in title:
            if tok in self.vectorizer.word2idx:
                text.append(self.vectorizer.word2idx[tok])
            else:
                text.append(3)

        input_variable = torch.LongTensor(text).view(1, -1)
        if self.use_cuda:
            input_variable = input_variable.cuda()

        input_lengths = [len(title)]

        text = []
        for tok in sec_seq:
            if tok in self.vectorizer.word2idx:
                text.append(self.vectorizer.word2idx[tok])
            else:
                text.append(3)

        prev_generated_seq = torch.LongTensor(text).view(1, -1)
        if self.use_cuda:
            prev_generated_seq = prev_generated_seq.cuda()

        outputs = []
        for i in range(num_exams):
            _, _, other = \
                self.model(input_variable, prev_generated_seq, input_lengths)
            length = other['length'][0]

            tgt_id_seq = [other['sequence'][di][0].item() for di in range(length)]
            tgt_seq = [self.vectorizer.idx2word[tok] for tok in tgt_id_seq]
            output = ' '.join([i for i in tgt_seq if i != '<PAD>' and i != '<EOS>' and i != '<SOS>'])
            outputs.append(output)
            prev_generated_seq = torch.LongTensor(tgt_id_seq).view(1, -1)
            if self.use_cuda:
                prev_generated_seq = prev_generated_seq.cuda()
        return outputs


#!/usr/bin/env python3
# -*- coding:utf-8 -*-
"""
@__Create Time__ = 2018/2/1 下午5:48
@__Description__ = " "
"""

from django.views.generic.list import ListView
from django.views.generic.edit import CreateView,BaseDeleteView,UpdateView,BaseUpdateView
from perm.mixins import PermissionRequiredMixin
from django.contrib.auth.mixins import LoginRequiredMixin
from ..models.group import AssetGroup
from ..models.asset import Asset
from ..forms.group import CreateAssetGroupForm,EditAssetGroupForm
from django.contrib import messages
from django.urls import reverse_lazy,reverse
from django.http import HttpResponseRedirect

class AssetGroupListView(LoginRequiredMixin,ListView):
    model = AssetGroup
    template_name = 'asset-group/list.html'

class CreateAssetGroupView(PermissionRequiredMixin,LoginRequiredMixin,CreateView):
    model = AssetGroup
    template_name = 'asset-group/create.html'
    form_class = CreateAssetGroupForm
    success_url = reverse_lazy('asset_group_list')
    permission_required = 'create_asset_group'

    def form_invalid(self, form):
        messages.error(self.request,form.errors.as_text())
        return super().form_invalid(form)

    def form_valid(self, form):
        asset_group_object = form.save(commit=False)
        asset_group_object.created_by = self.request.user.name
        return super().form_valid(form)

class DeleteAssetGroupView(PermissionRequiredMixin,LoginRequiredMixin,BaseDeleteView):
    model = AssetGroup
    success_url = reverse_lazy('asset_group_list')
    permission_required = 'delete_asset_group'

class EditAssetGroupView(PermissionRequiredMixin,LoginRequiredMixin,UpdateView):
    model = AssetGroup
    form_class = EditAssetGroupForm
    template_name = 'asset-group/create.html'
    success_url = reverse_lazy('asset_group_list')
    permission_required = 'edit_asset_group'


from datetime import datetime
from os.path import dirname, join

import pytest
from city_scrapers_core.constants import BOARD, PASSED, TENTATIVE
from city_scrapers_core.utils import file_response
from freezegun import freeze_time
from scrapy.settings import Settings

from city_scrapers.spiders.det_police_fire_retirement import (
    DetPoliceFireRetirementSpider,
)

test_response = file_response(
    join(dirname(__file__), "files", "det_police_fire_retirement.html"),
    url="http://www.rscd.org/member_resources/board_of_trustees/upcoming_meetings.php",
)
test_past_response = file_response(
    join(dirname(__file__), "files", "det_police_fire_retirement_past.html"),
    url="http://www.rscd.org/member_resources_/board_of_trustees/past_meeting_agendas___minutes.php",  # noqa
)

spider = DetPoliceFireRetirementSpider()
spider.settings = Settings(values={"CITY_SCRAPERS_ARCHIVE": False})

freezer = freeze_time("2019-04-05")
freezer.start()
spider._parse_past_documents(test_past_response)
parsed_items = [item for item in spider._parse_meetings(test_response)]
freezer.stop()

def test_total():
    assert len(parsed_items) == 38

def test_title():
    assert parsed_items[0]["title"] == "Board of Trustees"

def test_description():
    assert parsed_items[0]["description"] == ""

def test_start():
    assert parsed_items[0]["start"] == datetime(2019, 1, 10, 9, 0)
    assert parsed_items[-1]["start"].year < 2019

def test_end():
    assert parsed_items[0]["end"] is None

def test_id():
    assert (
        parsed_items[0]["id"]
        == "det_police_fire_retirement/201901100900/x/board_of_trustees"
    )

def test_status():
    assert parsed_items[0]["status"] == PASSED
    assert parsed_items[8]["status"] == TENTATIVE

def test_location():
    assert parsed_items[0]["location"] == {
        "name": "Retirement Systems Conference Room",
        "address": "500 Woodward Ave. Suite 300 Detroit, MI 48226",
    }
    assert parsed_items[-1]["location"] == {
        "name": "Retirement Systems",
        "address": "500 Woodward Ave. Suite 300 Detroit, MI 48226",
    }

def test_source():
    assert (
        parsed_items[0]["source"]
        == "http://www.rscd.org/member_resources/board_of_trustees/upcoming_meetings.php"  # noqa
    )

def test_links():
    assert parsed_items[0]["links"] == [
        {"href": "http://www.rscd.org/PFRS_3229A_01102019.pdf", "title": "Agenda"},
        {"href": "http://www.rscd.org/PFM_3229_011019.pdf", "title": "Minutes"},
    ]
    assert parsed_items[8]["links"] == []

def test_classification():
    assert parsed_items[0]["classification"] == BOARD

@pytest.mark.parametrize("item", parsed_items)
def test_all_day(item):
    assert item["all_day"] is False


# Our modules

# NumPy is a library for the Python programming language, adding support for large, multi-dimensional arrays and matrices, along with a large collection of high-level mathematical functions to operate on these arrays
import numpy as np
# Matplotlib is a plotting library for python and pyplot gives us a MatLab like plotting framework. We will use this in our plotter function to plot data.
import matplotlib.pyplot as plt
# Seaborn is a Python data visualization library based on matplotlib. It provides a high-level interface for drawing attractive and informative statistical graphics
import seaborn as sns
# Allows us to test parameters of classification algorithms and find the best one
from sklearn.model_selection import GridSearchCV
# Logistic Regression classification algorithm
from sklearn.linear_model import LogisticRegression
# Support Vector Machine classification algorithm
from sklearn.svm import SVC
# Decision Tree classification algorithm
from sklearn.tree import DecisionTreeClassifier
# K Nearest Neighbors classification algorithm
from sklearn.neighbors import KNeighborsClassifier
# Confusion Matrix classification
from sklearn.metrics import confusion_matrix

# ADGSTUDIOS 2021
class adgmodel:
    def plot_confusion_matrix(self, y_test, y_pred, title, truelbl, falselbl):
        try:
            print("")
            ticklabels = []
            ticklabels.append(truelbl)
            ticklabels.append(falselbl)
            cm = confusion_matrix(y_test, y_pred)
            ax = plt.subplot()
            sns.heatmap(cm, annot=True, ax=ax)
            ax.set_xlabel('Predicted labels')
            ax.set_ylabel('True labels')
            ax.set_title(title)
            ax.xaxis.set_ticklabels(ticklabels)
            ax.yaxis.set_ticklabels(ticklabels)
        except Exception as e:
            print('Error : ADGMLCLASS \n', e)

    def __logregcv(self, X_test, Y_test, X_train, Y_train, Parameters):
        if len(Parameters) > 0:
            parameters = Parameters
            pass
        else:
            parameters = {"C": [0.01, 0.1, 1], 'penalty': [
                'l2'], 'solver': ['lbfgs']}  # l1 lasso l2 ridge
            print("using parameters", parameters,
                  "Logistic Regression Algorithm")

        try:
            lr = LogisticRegression()
            gscv = GridSearchCV(lr, parameters, scoring='accuracy', cv=10)
            logreg_cv = gscv.fit(X_train, Y_train)
            print("tuned hyperparameters :(best parameters) ",
                  logreg_cv.best_params_)
            print("accuracy :", logreg_cv.best_score_)

            yhat = logreg_cv.predict(X_test)
            self.plot_confusion_matrix(
                Y_test, yhat, 'Confusion Matrix', '+', '-')
            return logreg_cv.best_score_
        except Exception as e:
            print('Error : ADGMLCLASS \n', e)

    def __svmcv(self, X_test, Y_test, X_train, Y_train, Parameters):
        if len(Parameters) > 0:
            parameters = Parameters
        else:

            parameters = {'kernel': ('linear', 'rbf', 'poly', 'rbf', 'sigmoid'),
                          'C': np.logspace(-3, 3, 5),
                          'gamma': np.logspace(-3, 3, 5)}

            print("using parameters", parameters,
                  "Support Vector Machine Algorithm")

        try:
            svm = SVC()
            gscv = GridSearchCV(svm, parameters, scoring='accuracy', cv=10)
            svm_cv = gscv.fit(X_train, Y_train)
            print("tuned hyperparameters :(best parameters) ", svm_cv.best_params_)
            print("accuracy :", svm_cv.best_score_)

            yhat = svm_cv.predict(X_test)
            self.plot_confusion_matrix(
                Y_test, yhat, 'Confusion Matrix', '+', '-')
            return svm_cv.best_score_

        except Exception as e:
            print('Error : ADGMLCLASS \n', e)

    def __tree(self, X_test, Y_test, X_train, Y_train, Parameters):
        if len(Parameters) > 0:
            parameters = Parameters
        else:
            parameters = {'criterion': ['gini', 'entropy'],
                          'splitter': ['best', 'random'],
                          'max_depth': [2*n for n in range(1, 10)],
                          'max_features': ['auto', 'sqrt'],
                          'min_samples_leaf': [1, 2, 4],
                          'min_samples_split': [2, 5, 10]}

            print("using parameters", parameters,
                  "Support Vector Machine Algorithm")
        try:
            tree = DecisionTreeClassifier()
            gscv = GridSearchCV(tree, parameters, scoring='accuracy', cv=10)
            tree_cv = gscv.fit(X_train, Y_train)

            print("tuned hyperparameters :(best parameters) ",
                  tree_cv.best_params_)
            print("accuracy :", tree_cv.best_score_)

            yhat = tree_cv.predict(X_test)
            self.plot_confusion_matrix(
                Y_test, yhat, 'Confusion Matrix', '+', '-')
            return tree_cv.best_score_
        except Exception as e:
            print('Error : ADGMLCLASS \n', e)

    def __knn(self, X_test, Y_test, X_train, Y_train, Parameters):
            if len(Parameters) > 0:
                parameters = Parameters
                pass
            else:
                parameters = {'n_neighbors': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
                              'algorithm': ['auto', 'ball_tree', 'kd_tree', 'brute'],
                              'p': [1, 2]}

                print("using parameters", parameters, "KNN Algorithm")

            try:
                KNN = KNeighborsClassifier()
                gscv = GridSearchCV(KNN, parameters, scoring='accuracy', cv=10)
                knn_cv = gscv.fit(X_train, Y_train)

                print("tuned hyperparameters :(best parameters) ",
                      knn_cv.best_params_)
                print("accuracy :", knn_cv.best_score_)
                yhat = knn_cv.predict(X_test)
                self.plot_confusion_matrix(
                    Y_test, yhat, 'Confusion Matrix', '+', '-')
                return knn_cv.best_score_
            except Exception as e:
                print('Error : ADGMLCLASS \n', e)

    def Train(self, Model, X_test, Y_test, X_train, Y_train, Parameters):
        try:
            if Model == 'svm':
                self.__svmcv(X_test, Y_test,
                             X_train, Y_train, Parameters)
            if Model == 'tree':
                self.__tree(X_test, Y_test, X_train, Y_train, Parameters)
            if Model == 'knn':
                self.__knn(X_test, Y_test, X_train, Y_train, Parameters)
            if Model == 'LogisticRegression':
                self.__logregcv(X_test, Y_test,
                                X_train, Y_train, Parameters)

        except Exception as e:
            print('Error : ADGMLCLASS \n', e)
            print('\n')
            print('Current Models that are supported is this version is :',
                  'svm,tree,knn,LogisticRegression - Type it as a string when using the function')

    def FindBestModel(self, X_test, Y_test, X_train, Y_train):
        print(
            'Finding best model please wait using the default ADG Optimized ML Formula...')
        algorithms = {'KNN': self.__knn(X_test, Y_test, X_train, Y_train,''), 'Tree': self.__tree(X_test, Y_test, X_train, Y_train,''),
                      'LogisticRegression': self.__tree( X_test, Y_test, X_train, Y_train,''), 'SVM': self.__svmcv(X_test, Y_test, X_train, Y_train,'')}
        bestalgorithm = max(algorithms, key=algorithms.get)
        print('Best Algorithm is', bestalgorithm,
              'with a score of', algorithms[bestalgorithm])


#!/usr/bin/env python
"""
Install wagtailnews using setuptools
"""
from setuptools import setup

with open('wagtailnews/version.py', 'r') as f:
    version = None
    exec(f.read())

with open('README.rst', 'r') as f:
    readme = f.read()

# Documentation dependencies
documentation_extras = [
    'Sphinx>=1.4.6',
    'sphinx-autobuild>=0.5.2',
    'sphinx_rtd_theme>=0.1.8',
    'sphinxcontrib-spelling==2.1.1',
    'pyenchant==1.6.6',
]

setup(
    name='wagtailnews-collection',
    version=version,
    description='News / blog plugin for the Wagtail CMS, but with news items belonging to collections, enforcing permissions',
    long_description=readme,
    author='Taylor C. Richberger',
    author_email='tcr@absolute-performance.com',
    url='https://github.com/taywee/wagtailnews-collection/',

    install_requires=[
        'wagtail>=1.5',
    ],
    extras_require={
        'docs': documentation_extras
    },
    zip_safe=False,
    license='BSD License',
    packages=[
        'wagtailnews',
        'wagtailnews.templatetags',
        'wagtailnews.views',
        ],
    include_package_data=True,
    package_data={
        'wagtailnews': [
            'templates/**/*.html',
            'templates/**/*.js',
            'static/**/*.js',
            'static/**/*.html',
            ]
        },
    classifiers=[
        'Environment :: Web Environment',
        'Intended Audience :: Developers',
        'Operating System :: OS Independent',
        'Programming Language :: Python',
        'Programming Language :: Python :: 3',
        'Framework :: Django',
        'License :: OSI Approved :: BSD License',
    ],
)


#-*- coding:utf-8 -*-

#这是服务器的首页，即为生成服务web.app的地方

import logging;logging.basicConfig(level=logging.INFO)
import asyncio,os,time,json
import orm
from handler import COOKIE_NAME,cookie2user
from models import User
from jinja2 import Environment,FileSystemLoader
from aiohttp import web
from coroweb import get,add_routes,add_static
from config import configs

#初始化jinja2模板
def init_jinja2(app,**kw):
	logging.info('init jinja2...')
	options = dict(
		    autoescape = kw.get('autoescape',True),
		    block_start_string = kw.get('block_start_string','{%'),
		    block_end_string = kw.get('block_end_string','%}'),
		    variable_start_string = kw.get('variable_start_string','{{'),
		    variable_end_string = kw.get('variable_end_string','}}'),
		    auto_reload = kw.get('auto_reload',True)
		)
	path = kw.get('path',None)
	if path is None:
		path = os.path.join(os.path.dirname(os.path.abspath(__file__)),'templates')
	logging.info('set jinja2 template path: %s' %str(path))
	env = Environment(loader=FileSystemLoader(path),**options)
	filters = kw.get('filters',None)
	if filters is not None:
		for name, f in filters.items():
			env.filters[name] = f
	#给app增添'__templating__'属性，值为jinja2的env，可以通过调用env.get_template来获取模板
	app['__templating__'] = env

#middlewares拦截器
async def logger_factory(app,handler):
	async def logger(request):
		logging.info('Request: %s %s' %(request.method,request.path))
		return (await handler(request))
	return logger

#解析cookie将登陆用户绑定到request对象的拦截器
async def auth_factory(app,handler):
	async def auth(request):
		logging.info('check user: %s %s' %(request.method,request.path))
		#设置新的属性__user__
		request.__user__ = None
		#已经import handler，同样的COOKIE_NAME就可以拿到
		cookie_str = request.cookies.get(COOKIE_NAME)
		if cookie_str:
			#从cookie_str中拿到用户名
			user = await cookie2user(cookie_str)
			if user:
				logging.info('set current user: %s' %user.email)
				request.__user__ = user
		return (await handler(request))
	return auth

#这里解析的是request里的数据
async def data_factory(app,handler):
	async def parse_data(request):
		if request.method == 'POST':
			if request.content_type.startswith('application/json'):
				request.__data__ = await request.json()
				logging.info('request json: %s' %str(request.__data__))
			elif request.content_type.startswith('application/x-www-form-urlencoded'):
				request.__data__ = await request.post()
				logging.info('request form: %s ' %str(request.__data__))
		return (await handler(request))
	return parse_data

#对handler的返回结果进行处理，使其符合aiohttp的response格式
async def response_factory(app,handler):
	async def response(request):
		logging.info('Response handler...')
		r = await handler(request)
		if isinstance(r,web.StreamResponse):
			#StreamResponse是aiohttp的标准返回对象，不需要处理
			return r
		if isinstance(r,bytes):
			resp = web.Response(body = r)
			resp.content_type = 'application/octer-stream'
			return resp
		if isinstance(r,str):
			if r.startswith('redirect:'):
				return web.HTTPFound(r[9:])
			resp = web.Response(body = r.encode('utf-8'))
			resp.content_type='text/html;charset=utf-8'
			return resp
		if isinstance(r,dict):
			#在handler里用'__template__'属性来作为键指向返回对象（html模板
			template = r.get('__template__')
			if template is None:
				resp = web.Response(body = json.dumps(r,ensure_ascii = False,default = lambda o:o.__dict__).encode('utf-8'))
				resp.content_type = 'application/json;charset=utf-8'
				return resp
			else:
				resp = web.Response(body = app['__templating__'].get_template(template).render(**r).encode('utf-8'))
				resp.content_type = 'text/html;charset=utf-8'
				return resp
		if isinstance(r,int) and r >= 100 and r<600:
			return web.Response(r)
		if isinstance(r,tuple) and len(r) == 2:
			t,m = r
			if isinstance(t,int) and t>=100 and t<600:
				return web.Response(t,str(m))
		#default
		resp = web.Response(body = str(r).encode('utf-8'))
		resp.conent_type= 'text/plain;charset=utf-8'
		return resp
	return response

def datetime_filter(t):
	delta = int(time.time() - t)
	if delta <60:
		return u'1分钟前'
	if delta <3600:
		return u'%s分钟前' %(delta //60)
	if delta <86400:
		return u'%s小时前' %(delta//3600)
	if delta <604800:
		return u'%s天前' %(delta//86400)
	dt = datetime.formtimestamp(t)
	return u'%s年%s月%s日' %(dt.year,dt.month,dt.day)

async def init(loop):
	await orm.create_pool(loop = loop,**configs.db)
	app = web.Application(loop = loop,middlewares=[logger_factory,auth_factory,response_factory])
	init_jinja2(app,filters=dict(datetime = datetime_filter))
	add_routes(app,'handler')
	add_static(app)
	srv = await loop.create_server(app.make_handler(),'127.0.0.1',9000)
	logging.info('server started at http://127.0.0.1:9000...')
	return srv

loop = asyncio.get_event_loop()
loop.run_until_complete(init(loop))
loop.run_forever()


#!/usr/bin/env python3
# -*- coding: utf-8 -*-

from sqlalchemy import create_engine, Table, Column, Integer, Float, String, Date, MetaData
from sqlalchemy.orm import registry
from Weather import Weather

class AlchContext:
    def __init__(self, provider):               #dbcontext
        AlchContext.Engine = create_engine(provider)
        metadata = MetaData()

        AlchContext.w_table = Table('weather', metadata,
            Column('id', Integer, primary_key=True),
            Column('wday', String(3)),
            Column('date', Date),
            Column('description', String(100)),
            Column('tempMax', Integer),
            Column('tempMin', Integer),
            Column('wind', Integer),
            Column('winddir', String(3)),
            Column('precip', Float),
            Column('humidity', Integer),
            Column('radiation', Integer)
        )
        metadata.create_all(AlchContext.Engine)
        mapper_reg = registry() # 2.0
        mapper_reg.map_imperatively(Weather, AlchContext.w_table)


from django.core.management.base import BaseCommand
from urllib.request import urlopen
from bs4 import BeautifulSoup
import json
from users.models import User
from wallets.models import WalletHistories
from binary.models import BinaryTree
import datetime

class Command(BaseCommand):
    help = "Count Binary Data"

    def handle(self, *args, **options):

	    def sendmatching(user, position, amount):
	        p = user
	        if user.upline_user_id != 'top':
	        	try:
	        		upline = BinaryTree.objects.get(user=str(user.upline_user_id))
	        		next_position = upline.position
	        		if upline.upline_user_id == user.upline_user_id:
	        			print('Max recursion se bacha le re baba'.format(upline, user))
	        			inpu = input("Enter y when done with resolution")
	        	except Exception as e:
	        		print("binary error {} at user {} and upline {}".format(e, str(user), str(user.upline_user_id)))
	        		upline = 'blank'

	        elif user.upline_user_id == 'top':
	            upline = 'blank'

	        if upline != 'blank':
	            p.rewarded = True
	            p.save()
	            try:
	                upline_user = User.objects.get(username=str(upline.user))
	                balance_before = upline_user.binary_income
	            except User.DoesNotExist:
	                print("user not found user is {}".format(str(upline.user)))
	                inputuser = input("Enter User:")
	                upline_user = User.objects.get(username=str(inputuser))
	                balance_before = upline_user.binary_income

	            direct_left = BinaryTree.objects.filter(direct_user_id=str(upline_user), position='left').count()
	            direct_right = BinaryTree.objects.filter(direct_user_id=str(upline_user), position='right').count()

	            if upline_user.total_users_left == 0 or upline_user.total_users_left == 0:
	                if position == 'left':
	                    upline_user.total_users_left += 1
	                    upline_user.left_side_business += 885
	                    if upline_user.total_users_right >= upline_user.total_users_left:
	                        if direct_left >= 1 and direct_right >= 1:
	                            upline_user.total_income += 200
	                            print('if1')
	                        upline_user.rank = 'Ellite'
	                else:
	                    upline_user.total_users_right += 1
	                    upline_user.right_side_business += 885
	                    if upline_user.total_users_left >= upline_user.total_users_right:
	                        if direct_left >= 1 and direct_right >= 1:
	                            upline_user.total_income += 200
	                            print('if2')
	                        upline_user.rank = 'Ellite'
	            else:
	                if position == 'left':
	                    upline_user.total_users_left += 1
	                    upline_user.left_side_business += 885
	                    if upline_user.total_users_left == upline_user.total_users_right:
	                        upline_user.total_income += 200
	                        print('if3')
	                        upline_user.rank = 'Ellite'
	                else:
	                    upline_user.total_users_right += 1
	                    upline_user.right_side_business += 885
	                    if upline_user.total_users_left == upline_user.total_users_right:
	                        upline_user.total_income += 200
	                        print('if4')
	                        upline_user.rank = 'Ellite'
	            upline_user.save()
	            sendmatching(upline, next_position, amount)
	        else:
	            return 0

	    newids = BinaryTree.objects.all()
	    self.stdout.write('{} jobs to complete'.format(newids.count()))
	    level = 0
	    for idv in newids:
	    	level += 1
	    	position = idv.position
	    	amount = idv.amount
	    	sendmatching(idv, position, amount)
	    	self.stdout.write('{} job/s completed'.format(level))

	    self.stdout.write( 'job complete' )