#!/usr/bin/python
from __future__ import print_function
from builtins import str
import os.path
import os
import fileinput
import pyjq
import sys
import json
import filelock
import shutil
import portalocker
import argparse

#invalid character for tags, (loosely) borrowed from TagSpaces requirements, but allowing for underscores:
invalidTagChars = " \"',:/\\|<>	"

def trimPath(filename):
	return filename.rstrip('/\\')

def getRealPath(filepath):
	if (os.path.islink(filepath)):
		filepath = os.path.realpath(filepath)
	return filepath

def getMetaDir(filename):
	return os.path.join(os.path.dirname(trimPath(filename)), '.ts')

def getMetaFileName(filePath):
	filePath = getRealPath(trimPath(filePath))
	return os.path.join(getMetaDir(filePath), os.path.basename(filePath) + ".json")

def getParentFile(metafile):
	metafile = getRealPath(trimPath(metafile))

	(metafolder, json) = os.path.split(metafile)
	#print "Meta folder after split: " + metafolder
	#print "Json file after split: "+ json
	#print " New path will be: " + metafolder[:-4] + "/" + json[:-5]
	#print "	Metafolder: " +metafolder
	#print "	metafolder[:-4]"  + metafolder[:-4]
	filePath = ''

	(parent, ts) = os.path.split(metafolder)

	if ts == ".ts" and json.endswith(".json"):
		filePath = os.path.join(parent, json[:-5])

	#print "new Filepath IS: " + filePath
	return filePath

#Public wrapper for private __writeFile method
def write(filename, content):
	return __writeFile(getRealPath(filename), content)

#Attempts to make directory, if one does not exist. Returns false only if requested directory is not usable.
def checkDir(dir):
	return __checkMakeDir(dir)

def getLockFileName(filename):
	filename = getRealPath(trimPath(filename))
	return os.path.join(getMetaDir(filename), os.path.basename(filename) + ".lock")

## A simple MyTags-specific lock on handling file operations for given file.
## Creates a lockfile in the metafolder of the file's parent named <filename>.lock
## This can be subsequently used with "with" statement block, or manually by using acquire() and release()
## "filename" should only be the full pathname of the main file, NOT the (.json) metafile in the metafolder.
def getLockFile(filename):
	filename = getRealPath(trimPath(filename))
	lock = None

	if __checkMakeDir(getMetaDir(filename)):
		lock = filelock.FileLock(getLockFileName(filename))

	#lock.filename = filename
	return lock

def __checkMakeDir(dir):
	try:
		os.makedirs(dir)
	except OSError as e:
		#print "checkDir failed for " +dir+ ": " +str(e)
		pass
	return os.path.isdir(dir)

def __readFile(filename, uselock=True):
	filename = getRealPath(filename)
	content = ''
	try:
		if (uselock):
			with portalocker.Lock(filename, mode="r", flags=1) as myfile:
				content = myfile.read()
		else:
			with open(filename) as myfile:
				content = myfile.read()

	except (IOError, portalocker.exceptions.BaseLockException) as e:
		print(e)
		pass
	return content

# Utility function that writes to a file. Makes the parent directory if it does not exist.
# Returns true if all went well. Returns false if could not access parent directorty or if there was an IOError. Print error to console.
# Does not check or use the getLockFile() method, since __writeFile may be used for any file, including the .json files. getLockFile()
# should be used by the code which calls __writeFile
def __writeFile(filename, content):
	filename = getRealPath(filename)
	if __checkMakeDir(os.path.dirname(filename)):
		try:
			with portalocker.Lock(filename, "w") as f:
				f.write(content)
		except (IOError, portalocker.exceptions.BaseLockException) as e:
			print(e)
			return False
	return True

def __generateJSON(tags):
	content = '{"tags":['
	firstTag = True
	for t in tags:
		if not firstTag:
			content +=','
		else: # @first Tag, allow for next tag to have comma...
			firstTag = False
		content += '{"title":"' + t.lower() + '","type":"sidecar"}'

	content += '],"appName":"MyTags"}'
	return content

def isValidTag(tag):
	#no leading or trailing whitespace!
	if len(tag.strip()) != len(tag):
		return 0

	for c in invalidTagChars:
		if c in tag:
			return 0
	return 1

#returns a tuple of two lists: (validtags, invalidtags)
def checkTags(tags):
	validTags = []
	invalidTags = []
	for t in tags:
		if (isValidTag(t)):
			validTags.insert(0,t)
		else:
			invalidTags.insert(0,t)
	return (validTags, invalidTags)

def __getTagsFromData(data, raw=True):
	if (raw):
		data=data.replace('\n', '')

	tags = []
	try:
		json.loads(data)
		rawtags = pyjq.all(data +  '| .tags[] | .title')
		for t in rawtags:
			tags.append(t.lower())

	except ValueError as e:
		pass#print "getTags ERROR for file ("+filename+") caught pyjq value error using data: \n" + data + "\n"
		#print e
	return tags

def getTags(filename, uselock=True):
	filename = getRealPath(filename)

	tags = []
	if (os.path.islink(filename)):
		filename = os.path.realpath(filename)

	metafile = getMetaFileName(filename)

	if (uselock):
		lockFile = getLockFile(filename)
		if (lockFile and os.path.isfile(metafile)):
			lockFile.timeout = 5

			with lockFile: #lock for exclusive use of file:
				tags = __getTagsFromData(__readFile(metafile))
	elif os.path.exists(metafile):
		tags = __getTagsFromData(__readFile(metafile, uselock))

	return tags

##
# addTags: adds tags to files
#		@input:
#			-filename
#			-tags - list of tags. Cannot contain invalid characters (see isValidTag() function above)
# 		@result: all of the tags are added to the given file's  .json metafile.
#			The contents of that metafile, if it already exists, will be overwritten, but whatever tags were
#			already in it will be retained (though, perhaps, in slightly reformatted JSON code, as another program
#			like TagSpaces may have created the original). Thus, after this function call, subsequent calls to "getTags"
#			for the given filename would contain all of its previous tags, plus any new ones passed to addTags.
#			Any tags pre-existing in the .json file not be duplicated by subsequent calls to addTags():
#			each tag is unique, and stored exactly once.
#		@returns: (success, failedTag), where success(boolean) is true if operation succeeded; false o/w.
#				if success == false, failedTag contains first invalid tag which caused the failure; or None if it failed because of failed file write error
#				if success == true, failedFiles is empty.
##
def addTags(filename, tags):
	filename = getRealPath(filename)
	metafile = getMetaFileName(filename)

	for t in tags:
		if not isValidTag(t):
			return (False, t)

	lockFile = getLockFile(filename)

	if lockFile: #found file, ready to try and lock:
		with lockFile: #got lock
			try:
				mode = "r+"
				if not os.path.exists(metafile):
					mode = "a+"
				with portalocker.Lock(metafile, mode) as mFile:
					mFile.seek(0)
					combinedTags = set(__getTagsFromData(mFile.read())) | set(tags)
					content = __generateJSON(combinedTags)
					mFile.seek(0)
					mFile.truncate()
					mFile.write(content)
					mFile.seek(0)
				#	print "addTAgs wrote content to " + metafile + ", new content is: " + mFile.read()
			except portalocker.LockException as e:
				print(e)
				return (False, None)
	else: # main file didn't exist
		#print "addTags did nothing, cause file didn't exist"
		return (False, None)

	#print "Successfully completed addTags()"
	return (True, None)

def addTagsBulk(filenames, tags):
	failedFiles = []
	for f in filenames:
		f = getRealPath(f)
		(success, failedTags) = addTags(f, tags)
		if not success:
			failedFiles.insert(0, f)

	return (not failedFiles, failedFiles)

def removeAllTags(filename):
	filename = getRealPath(filename)
	## Create bare json file with  empty dictionary "appName":"MyTags"
	# consider another option: just delete the metafile!
	result = False
	lockFile = getLockFile(filename);

	with lockFile:
		result = __writeFile(getMetaFileName(filename), __generateJSON([]))
	return result

def removeSomeTags(filename, removeTags):
	filename = getRealPath(filename)
	# Attempts to remove the given tags from the metafile of the given file.
	# Returns: True if file exists and it no longer as any of the tags from 'removeTags' in its metafile (if metafile exists).
	# 		   False otherwise.
	# NB my coding choice: do we ensure "atomic" operation in which we open read/write, parse tags, and the write with modified output?
	#               this would re-write to the file even if we don't need to, thus updating the time-stamp as if the tags changed, even if they didn't
	#               On the other hand, if we first open in read-only mode, then check for tags and write only if tags changed, there is the possibility
	#				that one or the file will change between reading and writing. Thus, always opening in R/W mode seems best, thus always writing to file.
	#

	#print "Removesometags("+filename+", ["+",".join(removeTags)+"])"
	tags = set()
	metafile = getMetaFileName(filename)
	lock = getLockFile(filename)
	if (lock):
		with lock:
			if not os.path.exists(filename):
				print("WHOA! removeSomeTags(" + filename + ") could not find the file!")
				return False

			#print "removeSomeTags("+filename+")...\n"
			try:
				if (os.path.isfile(metafile)):
					with portalocker.Lock(metafile, 'r+') as myfile:
						#print "\nData :\n" + data +"\n<\enddata>"
						tags = set(__getTagsFromData(myfile.read()))

						for t in removeTags:
							#print "removing " + t + " from set ([" + ",".join(tags) +"])"
							tags.discard(t)
							#print "now have: ([" + ",".join(tags) +"])"
						output = __generateJSON(tags)

						myfile.seek(0)
						myfile.truncate()
						myfile.write(output)
						#print "Wrote to :" + metafile  + " : " + __generateJSON(tags)
			except IOError as e:
				print("WHOA! removeSomeTags(" + filename + ") threw exception:")
				print(e)
				return False
			#print os.path.isfile(filename)
			#print "End of RemovesomeTags(" +filename+")...closed. Here's getTags():" + "|".join(getTags(filename))
	return True

''' updateTags: removes all tags from, and then add the input tags
	return (success, tag): success == true if all went well; false if there were invalid tag, the first of which is also returned
	Throws Exception if lock or IO error occured.
'''
def replaceTags(filename, tags):
	filename = getRealPath(filename)
	for t in tags:
		if not isValidTag(t):
			return (False, t)

	content = __generateJSON(tags)
	success = False
	lockFile = getLockFile(filename);

	with lockFile:
		metafile = getMetaFileName(filename)
		success = __writeFile(metafile, content)

	return (success, None)

''' Checks src, dest, based on whether they exist, and whether they are files or dirs.
	Returns  a tuple (success, dest, reason): success is True if src and dest
	are a valid pair for copying/moving with the given safe mode (safe=True means no files
	can be overwritten.  Dest is the fullPath of the (potentially updated) destination.
	It is an empty string ('') if the files cannot be copied/moved, and,
	if this is the case, 'reason' contains a message about why this is which can be
	used for loggin.
'''
def __checkSrcDest(src,dest, safe=True, hardLink=False):
	src = getRealPath(src)

	success = True
	reason = ''
	src = trimPath(src)
	dest = trimPath(dest)
	if (src == dest):
		success = False
		reason = "Src and Dest are the same!"
	elif (not os.path.exists(src)):
		reason = "Source does not exist"
		success = False
	elif (not os.path.exists(dest)): # dest does not yet exist; much check parent:
		#print "\nDest does NOT exist!!! " + dest
		success = checkDir(os.path.dirname(dest))
		if (not success):
			reason = "Could not access parent"
	elif (os.path.isfile(dest) and (safe or hardLink or os.path.isdir(src))): #
		if (os.path.exists(dest)):
			pass #print "The following dest exists: " +dest
		reason = "Cannot move src-file["+src+"] to existing dest file ["+dest+"]: either safe mode is on, or src is dir!"
		success = False
	elif (os.path.isdir(dest)): # src moved to be child of dest.
		if not checkDir(dest):
			success = False
			reason = "Cannot access dest/parent dir"
		else: #move dest to explicit child/sub-folder
			dest = os.path.join(dest, os.path.basename(src))
		#	print "changing dest to child; now it is: " + dest
			if (os.path.exists(dest)): #dest actually exists!
				if (os.path.isdir(src)):#dest already is a sub-dir
					success = False
					reason = "sub-directory by this name already exists; cannot move src to it"
				elif safe: #file by this name already exists, but safe mode is on:
					success = False
					reason = "trying to move into dest folder, but file by this name alreadt exists"
	#else:
	#	print("checkSrcDest() found no errors. Returning success...")
	return (success, dest, reason)

# Move a file, and its sidecar file, to the destination.
# Input:	srcFilePath - the full path and filename of the file to move.
#			destFilePath - the full path and filename of the new location for the file
#			safe - If true (default), will not overwrite the destFilePath if its exist;
#				   in such a case does nothing and returns False.
# Return:	True if file moved successfuly. This means the srcFilePath no longer exists, but the same file is now at destFilePath.
#		    Its sidecar file would now be in the .ts directory of its new parent directory, renamed to match the new filename.
# 			False if it did not but did not throw error. OSError Exception will be caught and not re-raised. Other exceptions are not caught.
# Errors:   Any Exceptions besides an OSError will be raised/passed to the calling block, and not handled/trapped here.
def moveFile(src, dest, safe=True):
	src = getRealPath(src)
	#print "inside moveFile(" +src +"," +dest
	src = trimPath(src)
	dest = trimPath(dest)

	(success, dest, reason) = __checkSrcDest(src, dest, safe)

	srcMeta = getMetaFileName(src)
	destMeta = getMetaFileName(dest)

	if success:
		lock1 = getLockFile(src)
		with lock1:
			lock2 = getLockFile(dest)
			with lock2:
				try:
					if os.path.isdir(src):# working with a dir; not using file locks:
						shutil.move(src, dest)
						if (os.path.exists(srcMeta) and checkDir(getMetaDir(dest))):
							shutil.move(srcMeta, destMeta)
					else: #working with files; using file-locks:
						with portalocker.Lock(dest, "w") as d:
							with portalocker.Lock(src, "r") as s:
							#print "Got all the locks we need! Trying to rename..."
								shutil.move(src, dest)
								if (os.path.exists(srcMeta) and checkDir(getMetaDir(dest))):
									shutil.move(srcMeta, destMeta)

				except  (OSError, shutil.Error) as e:
					print("\nCaught error in movefile() function:\n")
					#print e
					reason = "caught error in renaming: "
					reason += str(e)
					success = False

	#print "moving file to "+  dest + " returning " + str(success and os.path.exists(dest))
	#if (reason):
		#print " because: " +reason
	return dest if success else ''

# Rename a file, in its parent directory. Will not overwrite existing files, but will raise Exception if src already exists.
# Input:	srcFilePath - the full path and filename of the file to move.
#			newname - the new *name* of the file, not including its path or parent.
# Return:	True if file copied successfuly
# False:	False if it did not but did not throw error.
# Errors:   Any Exceptions (e.g., IOError) will be raised/passed to the calling block, and not handled/trapped here.
def renameFile(srcFilePath, newname):
	srcFilePath = getRealPath(srcFilePath)
	return moveFile(srcFilePath, os.path.join(os.path.dirname(srcFilePath), newname))

# Copies a file, and its sidecar file (if it exists), to the destination.
# Input:	srcFilePath - the full path and filename of the file to move.
#			destFilePath - either the full path of the directory to copy the file, or the full path of the new filename.
#			safe - If true (default), will not overwrite the destFilePath if its exist;
#				   in such a case does nothing and returns False.
#			hardLink: If true (default: false), function will attempt to create a hardlink at src, instead of copy.
# 				Only matters if src is a file, not a dir.
# Return:	True if file copied successfuly
# False:	False if it did not but did not throw error.
# Errors:   Any Exceptions (e.g., IOError) will be raised/passed to the calling block, and not handled/trapped here.
def copyFile(src, dest, safe=True,hardLink=False):
	src = trimPath(getRealPath(src))
	#src = trimPath(src)
	dest = trimPath(dest)

	(success, dest, reason) = __checkSrcDest(src, dest, safe, hardLink)

	srcMeta = getMetaFileName(src)
	destMeta = getMetaFileName(dest)

	if success:
		lock1 = getLockFile(src)
		with lock1:
			lock2 = getLockFile(dest)
			with lock2:
				try:
					if os.path.isdir(src):# working with a dir, using shutil.copytree() instead of copy()
						shutil.copytree(src, dest)
						if (os.path.exists(srcMeta) and checkDir(getMetaDir(dest))):
							shutil.copy2(srcMeta, destMeta)
					else: #working with files; using file-locks:
						if (hardLink): # hardlink: use os.link(). Must check if dest is dir:
							if( os.path.isdir(dest)): # copying one file into another folder
								originalDest = dest
								dest = os.path.join(dest, os.path.basename(src))
								print("Had to turn dest[" + originalDest + "] folder into full path of filename [" + dest + "]")
								# give dest a filename instead of dir, to avoid hardLink problem.
								if (os.path.exists(dest)):
									print ("Hmmm...new dest already exists!")
							with portalocker.Lock(src, "r") as s: # cannot use portraLocker on dest with os.link()
							#print "Got all the locks we need! Trying to copy..."
								#print ("trying HARDLINK1..." + src + " -> " + dest)
								#if (os.path.exists(dest)):
								#		print "But dest [" + dest + "] already exists! This should not have happened..."
								os.link(src,dest)
								#	print "made HARDLINK1..."
						else: #no hardlink. doing the normal stuff...
							with portalocker.Lock(dest, "w") as d:
								with portalocker.Lock(src, "r") as s:
								#print "Got all the locks we need! Trying to copy..."
									shutil.copy2(src, dest)
						#copy meta files;
						if (os.path.exists(srcMeta) and checkDir(getMetaDir(dest))):
							if (hardLink): # try making link!
								#print "trying HARDLINK2..."
								os.link(srcMeta, destMeta)
								#print "made HARDLINK2..."
							else:
								shutil.copy2(srcMeta, destMeta)

				except  OSError as e:
					reason = "\nCaught error in copyfile() function:\n"
					reason += str(e)
					print(reason)
					success = False

	#print "Copying file to "+  dest + " returning " + str(success and os.path.exists(dest))
	#if (reason):
		#print " because: " +reason
	return dest if success else ''

def deleteFile(filename):
	filename = getRealPath(filename)
	lockFile = getLockFile(filename)
	with lockFile:
		os.remove(filename);
		metaF = getMetaFileName(filename)
		if (os.path.exists(metaF)):
			os.remove(metaF)
	return not os.path.exists(filename)

'''
	deleteFolder:
	Deletes a folder. Only works on an empty directory,
	unless recursive=True, in which case, it removes
	the directory and all of its contents.
	Does not catch any Exceptions, but if none are thrown,
	returns true if given folder no longer exists,
	false otherwise.
'''
def deleteFolder(foldername, recursive=False):
	foldername = getRealPath(foldername)
	if (recursive):
		shutil.rmtree(foldername)
	else:
		os.rmdir(foldername)
	success = not os.path.isdir(foldername)
	if (success):
		deleteMetaFile(foldername)
	return success

# Deletes the given file's sidecar file. Returns True if the given file no longer has a sidecar file.
# Returns False if sidecar file exists and it could not be removed.
def deleteMetaFile(filename):
	filename = getRealPath(filename)
	metafile = getMetaFileName(filename)
	lockFile = getLockFile(filename)
	with lockFile:
		if (os.path.exists(metafile)):
			os.remove(metafile)
	return not os.path.exists(metafile)

def cleanMetaFolder(parentdir):
	parentdir = getRealPath(parentdir)
	metadir = getMetaDir(os.path.join(parentdir, "filename"))
	#print "In cleanMetaFolder("+parentdir+"):\n"
	try:
		for f in findOrphanMetaFiles(parentdir):
			parentFile = os.path.join(parentdir,os.path.splitext(os.path.basename(f))[0])

			if (not os.path.exists(parentFile)):
				#print "Going to remove " + f
				os.remove(f)

			else:
				print("Not going to remove "+ f)
		removeUnusedLocks(metadir)
	except (OSError, IOError) as e:
		print("CleanMetaFoldr(" +parentdir+") caught exception: " + str(e))

# Returns list of .JSON metafiles (not parent files!) in the .ts folder of given "directory" that have no corresponding file in "directory'
def findOrphanMetaFiles(directory):
	directory = getRealPath(directory)
	directory = trimPath(directory)
	metadir = getMetaDir(os.path.join(directory, "filename"))
	orphanedMetaFiles = []

	if (os.path.isdir(metadir)):
		for f in os.listdir(metadir):
			(root, ext) = os.path.splitext(f)
			parentFile = os.path.join(directory, root)
			if (ext == ".json") and (not os.path.exists(parentFile)): # orphaned metaFile!
				orphanedMetaFiles.insert(0,os.path.join(metadir, f))
				#print "Found orphaned metafile: " + os.path.join(metadir, f)  + "\n"

	return orphanedMetaFiles

def removeUnusedLocks(metadir):
	metadir = getRealPath(metadir)
	#print "removeUnusedLocks("+metadir+"):\n"
	if (os.path.isdir(metadir)):
		for f in os.listdir(metadir):
			print("checking " + f)
			(root, ext) = os.path.splitext(f)
			#print "root: " + root + ", ext = " + ext
			if ext == ".lock":
				parentFile = os.path.join(os.path.dirname(metadir), root)
				#print "Checking if parent file " + parentFile + " exists..."
				if (not os.path.exists(parentFile)):

					#print "No! About to remove " + os.path.join(metadir, f)
					os.remove(os.path.join(metadir, f))
				else:
					#print "Yes! Not going to remove"
					pass

			else:
				pass
				#print "Ext ("+ext+") is not '.lock'"

def printTags(tags):
	if (tags):
		print(" ".join(tags))

def cl_getTags(args):
	printTags(getTags(args.filename, False))

def cl_addTags(args):
	addTags(args.filename,args.tags)

def cl_removeTags(args):
	removeSomeTags(args.filename, args.tags)

def cl_removeAllTags(args):
	removeAllTags(args.filename)

def cl_replaceTags(args):
	replaceTags(args.filename, args.tags)

def commandLine():
	#print sys.argv
	import argparse

	parser = argparse.ArgumentParser(description='Manage file/folder tags in sidecar files (.ts)')
	#parser.add_argument('command', metavar='[command]', type=int, nargs='+', help='an integer for the accumulator')
	subparsers = parser.add_subparsers(help='[command] help')
	parser_tags = subparsers.add_parser('tags', help='tags help')
	parser_tags.add_argument('filename', metavar='Filename', type=str, help='Filename')
	parser_tags.set_defaults(func=cl_getTags)
	#parser_get.add_argument('tag', metavar='tag', type=str, nargs = '+', help='tag')

	# add tags command:
	parser_add = subparsers.add_parser('add', help='add help')
	parser_add.add_argument('filename', metavar='Filename', type=str, help='Filename')
	parser_add.add_argument('tags', metavar="tag", type=str, nargs="+", help='Tags')
	parser_add.set_defaults(func=cl_addTags)

	#remove tags command:
	parser_remove = subparsers.add_parser('remove', help='remove help')
	parser_remove.add_argument('filename', metavar='Filename', type=str, help='Filename')
	parser_remove.add_argument('tags', metavar="tag", type=str, nargs="+", help='Tags')
	parser_remove.set_defaults(func=cl_removeTags)

	#removeALL tags command:
	parser_removeAll = subparsers.add_parser('remove-all', help='remove-all help')
	parser_removeAll.add_argument('filename', metavar='Filename', type=str, help='Filename')
	parser_removeAll.set_defaults(func=cl_removeAllTags)

	#replace tags command:
	parser_replace = subparsers.add_parser('replace', help='replace help')
	parser_replace.add_argument('filename', metavar='Filename', type=str, help='Filename')
	parser_replace.add_argument('tags', metavar="tag", type=str, nargs="+", help='Tags')
	parser_replace.set_defaults(func=cl_replaceTags)

	args = parser.parse_args()
	args.func(args)

if __name__ == "__main__":
	commandLine()


import asyncio, time, re, os, requests, json, random, datetime
from lxml import etree
from multiprocessing.dummy import Pool
from pymongo import MongoClient
from PIL import Image
from tool.ImgOut import iop

client = MongoClient('localhost', 27017)
db = client.MMjpg
collection1 = db.Image1
collection2 = db.Image2
collection3 = db.Image3
collection4 = db.Image4
set_path = 'C:/Users/Administrator/Desktop/图/fhx20161004'
out_path = 'C:/Users/Administrator/Desktop'

def random_output(output_path, source_collection):
    st = time.time()
    img_list = source_collection.find()
    img_random = img_list[random.randint(0, img_list.count())]
    img_set_name = img_random['set_name']
    img_set_list = source_collection.find({'set_name':img_set_name})
    if not os.path.exists(output_path + os.path.sep + img_set_name):
        os.mkdir(output_path + os.path.sep + img_set_name)
    for img_dict in img_set_list:
        save_path = output_path + os.path.sep + img_dict['set_name'] + os.path.sep + img_dict['img_num'] + '.' + img_dict['img_type']
        with open(save_path, 'wb') as img:
            img.write(img_dict['img_data'])
        img.close()
    print('输出完毕，耗时%s' % str(time.time()-st))
# iop.random_output(out_path, collection4)

def insert_pic(i, save_collection = collection4):
    img_name_list = os.listdir(set_path + os.path.sep + i)
    for each_img in img_name_list:
        if save_collection.find({'set_name': i, 'img_num': each_img.split('.')[0]}).count() == 0:
            start_time = time.time()
            with open(set_path + os.path.sep + i + os.path.sep + each_img, 'rb') as img:
                img_data = img.read()
            img.close()
            insert_dict = {}
            insert_dict['img_data'] = img_data
            insert_dict['set_name'] = i
            insert_dict['img_type'] = each_img.split('.')[-1]
            insert_dict['img_num'] = each_img.split('.')[0]
            save_collection.insert(insert_dict)
            print('录入一张图片，来自%s的第%s张图片，耗时%s' % (insert_dict['set_name'], insert_dict['img_num'], str(time.time() - start_time)))

def up_photos():
    collection = [collection1, collection2, collection3, collection4]
    save_file = out_path + os.sep + 'photos'
    for i in collection:
        pic_list = i.find()
        collection_count = 0
        while True:
            try:
                save_pic_count = len(os.listdir(save_file))
                save_path = save_file + os.sep + 'img_%s.' % str(save_pic_count + 1) + pic_list[collection_count]['img_type']
                with open(save_path, 'wb') as pic:
                    pic.write(pic_list[collection_count]['img_data'])
                pic.close()
                collection_count += 2
            except:
                break

up_photos()
# exit()

# b = requests.get('http://imgs.aixifan.com/live/1493028073796/1493028073796.jpg').content
#
# fl = open('a.txt', 'wb')
# fl.write(b)
# fl.close()
# fl = open('a.txt', 'rb')
# with open('b.jpg', 'wb') as c:
#     c.write(fl.read())
# c.close()


import asyncio

host = '127.0.0.1'
port = 8889
global CLIENTS
CLIENTS = {}

async def handle_client_msg(reader, writer):
    addr = writer.get_extra_info('peername')
    print(f"New client: {addr[0]}:{addr[1]}")

    data = await reader.read(1024)
    message = data.decode()

    pseudo = message.split("|")[1]
    print(f"{addr[0]}:{addr[1]} has chosen the pseudo {pseudo}")
    if addr not in CLIENTS:
        CLIENTS[addr] = {}
        CLIENTS[addr]["r"] = reader
        CLIENTS[addr]["w"] = writer
        CLIENTS[addr]["pseudo"] = pseudo
        await announce(f"{pseudo} has joined the chatroom")

    while True:
        data = await reader.read(1024)
        if data == b'':
            break

        msg = data.decode()
        sender_pseudo = CLIENTS[addr]["pseudo"]
        formatted_msg = f"{sender_pseudo}: {msg}"
        print(formatted_msg)

        for client_addr, client_info in CLIENTS.items():
            if client_addr != addr:
                client_info["w"].write(formatted_msg.encode())
                await client_info["w"].drain()

        if addr in CLIENTS and "saluted" not in CLIENTS[addr]:
            await writer.drain()
            CLIENTS[addr]["saluted"] = True

    print(f"Connection with {pseudo} closed")
    writer.close()

async def announce(announcement):
    for client_addr, client_info in CLIENTS.items():
        announcement = f"{announcement}"
        client_info["w"].write(f"Announcement: {announcement}".encode())
        await client_info["w"].drain()

async def main():
    server = await asyncio.start_server(handle_client_msg, host, port)
    addrs = ', '.join(str(sock.getsockname()) for sock in server.sockets)
    print(f'Serving on {addrs}')

    async with server:
        await server.serve_forever()

if __name__ == "__main__":

    asyncio.run(main())


# source venv/bin/activate

import os
from farcaster import Warpcast
from dotenv import load_dotenv
import requests
from eth_account import messages, Account
import time
import re

load_dotenv()

client = Warpcast(mnemonic=os.environ.get("FARC_PRIVKEY"))
mnemonic = os.environ.get("KIWI_MNEMONIC")

def main():
	for cast in client.stream_casts(skip_existing=True):
		if cast.text.startswith("@postkiwi"):
			# print(cast.text)
			parent = client.get_cast(cast.parent_hash)
			# print(parent)
			link = extract_link(parent.cast.text)
			if link is None and parent.cast.embeds is not None:
				link = parent.cast.embeds.images[0].sourceUrl
				print(link)
			elif link is None and parent.cast.embeds is None:
				continue

			# print(link)
			response = send_to_kiwinews(cast.text[10:], link)

			if response.status_code == 200:
				print(response.json())
				# _ = client.post_cast(text="Added 🥝", parent=cast.hash)
				_ = client.like_cast(cast.hash)
			else:
				print('Error: {}'.format(response.text))

def send_to_kiwinews(title, href):
	url = "https://news.kiwistand.com/api/v1/messages"

	request_body = {
		"timestamp": int(time.time()),
		"type": "amplify",
		"title": title,
		"href": href,
	}

	message = {
		"types": {
			"EIP712Domain": [
				{"name": "name", "type": "string"},
				{"name": "version", "type": "string"},
				{"name": "salt", "type": "bytes32"},
			],
			"Message": [
				{"name": "title", "type": "string"},
				{"name": "href", "type": "string"},
				{"name": "type", "type": "string"},
				{"name": "timestamp", "type": "uint256"},
			]
		},
		"primaryType": "Message",
		"domain": {
			"name": "kiwinews",
			"version": "1.0.0",
			"salt": bytes.fromhex("fe7a9d68e99b6942bb3a36178b251da8bd061c20ed1e795207ae97183b590e5b"),
		},
		"message": request_body
	}

	acct = Account.from_mnemonic(mnemonic)

	structured_msg = messages.encode_structured_data(message)
	# print(structured_msg.body.hex())

	signature = Account.sign_message(structured_msg, acct.key)

	request_body["signature"] = signature.signature.hex()

	response = requests.post(url, json=request_body)

	return response

def extract_link(text):
	url_pattern = re.compile(r'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\\(\\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+')
	urls = re.findall(url_pattern, text)

	return urls[0] if urls else None

main()


import h5py
import numpy as np
import os, glob
import cartopy.crs as ccrs
import cartopy
import matplotlib.pyplot as plt
import matplotlib.path as mpath
def show_plot(ax1):
	ax1.set_extent([-180, 180, -90, -65], ccrs.PlateCarree())
	ax1.add_feature(cartopy.feature.LAND)
	ax1.add_feature(cartopy.feature.OCEAN)
	theta = np.linspace(0, 2*np.pi, 100)
	center, radius = [0.5, 0.5], 0.5
	verts = np.vstack([np.sin(theta), np.cos(theta)]).T
	circle = mpath.Path(verts * radius + center)
	ax1.set_boundary(circle, transform=ax1.transAxes)
	# plt.show()
	return

def plot_tracks(lon, lat, variable=None, **kwargs):
	# global ax1
	if variable is not None:
		plt.scatter(lon, lat, c=variable, transform = ccrs.PlateCarree(), **kwargs)
	else:
		plt.plot(lon, lat, transform = ccrs.PlateCarree(), **kwargs)
	return

#################################################
os.chdir("/home/jeffrey/Snowblower/Small_data/")
ls = glob.glob("*.h5")
f = h5py.File(ls[-1], 'r')
#################################################

fakelons = 180 - np.random.random(50000)*360
fakelats = (np.random.random(50000)*15+75)*-1
fake_var = (np.random.random(50000)*0.7+0.3)*fakelons

lon_range = np.arange(-180, 180, 0.75)
lat_range = np.arange(-60, -90, -0.75)

lon_inds = np.digitize(fakelons, lon_range)
lat_inds = np.digitize(fakelats, lat_range)

meaned_data = np.zeros((lon_range.size, lat_range.size))
std_data = np.zeros((lon_range.size, lat_range.size))
## indexed as [lon, lat]

for i in range(len(meaned_data)):
	for j in range(len(meaned_data[i])):

		grab_inds = np.argwhere((lat_inds == j)*(lon_inds==i))
		#do we want to set a test for nans? if indexes are empty
		meaned_data[i, j] = np.mean(fake_var[grab_inds])
		std_data[i, j] = np.std(fake_var[grab_inds])

fig = plt.figure(figsize=(11,8))

import matplotlib.gridspec as gridspec

# Create 2x2 sub plots
# gs = gridspec.GridSpec(2, 2)

gs0 = fig.add_gridspec(2, 1)

gs00 = gs0[0].subgridspec(1, 6)
gs01 = gs0[1].subgridspec(1, 6)

ax1 = fig.add_subplot(gs00[0,1:5], projection=ccrs.SouthPolarStereo())
# .subplot(2,2,(1,2), projection=ccrs.SouthPolarStereo())
plt.title("Raw data, with noise added")
plot_tracks(fakelons, fakelats, variable=fake_var, cmap='bwr')
cbar0 = plt.colorbar(ax=ax1)
cbar0.set_label("Raw longitude")
show_plot(ax1)

# ax2 = plt.subplot(223, projection=ccrs.SouthPolarStereo())
ax2 = fig.add_subplot(gs01[0,:3], projection=ccrs.SouthPolarStereo())

plt.title("Binned (mean) data")
data_crs = ccrs.PlateCarree()
xx, yy = np.meshgrid(lon_range, lat_range)
plt.pcolormesh(xx, yy, meaned_data.T, transform=data_crs, cmap='bwr')
cbar = plt.colorbar(ax=ax2)
cbar.set_label("Colorbar Label (unit)")
show_plot(ax2)

# ax3 = plt.subplot(224, projection=ccrs.SouthPolarStereo())
ax3 = fig.add_subplot(gs01[0,3:], projection=ccrs.SouthPolarStereo())
plt.pcolormesh(xx, yy, std_data.T, transform=data_crs)
cbar2 = plt.colorbar()
cbar2.set_label("Standard deviation")
show_plot(ax3)

plt.tight_layout()
plt.show()


from sacred import Experiment
import torch
import numpy as np
import pandas as pd
import os
import math
import warnings
import itertools
import numbers
import torch.utils.data as utils
#import matplotlib.pyplot as plt
from sklearn.metrics import roc_curve, auc, roc_auc_score, average_precision_score

import sys
sys.path.insert(0, '../src')
from data_loader_oo import DataContainer
from GRUD_model import grud_model

ex = Experiment("GRU-D-mean")
use_cuda = torch.cuda.is_available()
device = torch.device("cuda:0" if use_cuda else "cpu")

#TODO: data gets padded before, does this have any effect if i just take the last step output as output?

def fit(model, criterion, l2_penalty, learning_rate,\
        train_dataloader, dev_dataloader, test_dataloader,\
        learning_rate_decay=0, n_epochs=30, checkpoint_path = 'model_checkpoints', sacred_run=None):

    test_freq = int(len(train_dataloader) / 4)
    use_cuda = torch.cuda.is_available()
    device = torch.device("cuda:0" if use_cuda else "cpu")

    # to check the update
    old_state_dict = {}
    for key in model.state_dict():
        old_state_dict[key] = model.state_dict()[key].clone()

    AU_PRCs = list()
    n_batch = 0
    for epoch in range(n_epochs):
        print("starting Epoch: {}".format(epoch))
        if learning_rate_decay != 0:

            # every [decay_step] epoch reduce the learning rate by half
            if  epoch % learning_rate_decay == 0:
                learning_rate = learning_rate/2
                optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate, weight_decay= l2_penalty)
                print('at epoch {} learning_rate is updated to {}'.format(epoch, learning_rate))

        # train the model
        losses, acc = [], []
        label, pred = [], []
        y_pred_col= []

        #get a minibatch
        for train_data, train_label, train_num_obs in train_dataloader:
            model.train()
            #push current sample to GPU or CPU
            train_data, train_label, train_num_obs = train_data.to(device), train_label.to(device), train_num_obs.to(device)

            # Zero the parameter gradients
            optimizer.zero_grad()

            # Forward pass : Compute predicted y by passing train data to the model
            #y_pred = model(train_data)[:,-1,:] #last output

            #with pad after:
            y_pred = model(train_data)
            #print(y_pred.size())
            y_pred = torch.squeeze(y_pred)
            #print(y_pred.size())
            y_pred = torch.gather(y_pred,1,train_num_obs.long())
            #print(y_pred.size())
            #print(train_num_obs.size())

            #print(train_num_obs)

            # Compute loss
            loss = criterion(y_pred, train_label)
            #print(loss)
            acc.append(
                torch.eq(
                    (torch.sigmoid(y_pred).data > 0.5).float(),
                    train_label)
            )
            losses.append(loss.item())
            print(str(n_batch)+" loss: "+ str(loss.item()) )
            #print(model.w_dg_x.weight)
            #print(model.w_dg_x.bias)

            # perform a backward pass, and update the weights.
            loss.backward()
            optimizer.step()

            #sacred_run.log_scalar('loss', loss, n_batch)
            #sacred_run.log_scalar('acc', acc, n_batch)
            n_batch = n_batch + 1

            if n_batch % test_freq == 0:
                print("Validate...")
                losses, acc = [], []
                label, pred = np.array([]), np.array([])
                model.eval()

                for dev_data, dev_label, dev_num_obs in dev_dataloader:

                    dev_data, dev_label, dev_num_obs = dev_data.to(device), dev_label.to(device), dev_num_obs.to(device)

                    print(dev_data.size())
                    print(dev_label.size())
                    print(dev_num_obs.size())

                    optimizer.zero_grad()
                    # Forward pass : Compute predicted y by passing train data to the model

                    #y_pred = model(dev_data)[:,-1,:] #last output
                    y_pred = model(dev_data)
                    y_pred = torch.squeeze(y_pred)
                    #print(y_pred.size())
                    y_pred = torch.gather(y_pred,1,dev_num_obs.long())

                    # Compute loss
                    loss = criterion(y_pred, dev_label)
                    acc.append(
                        torch.eq(
                            (torch.sigmoid(y_pred).data > 0.5).float(),
                            dev_label)
                    )
                    losses.append(loss.item())

                    label = np.append(label, dev_label.detach().cpu().numpy())
                    pred = np.append(pred, y_pred.detach().cpu().numpy())

                dev_acc = torch.mean(torch.cat(acc).float())
                dev_loss = np.mean(losses)

                dev_pred_out = pred
                dev_label_out = label

                va_auc = roc_auc_score(label, pred)
                va_prc = average_precision_score(label, pred)
                print("validation auc:{}".format(va_auc))
                print("validation prc:{}".format(va_prc))

                sacred_run.log_scalar('va_auc', va_auc, n_batch)
                sacred_run.log_scalar('va_prc', va_prc, n_batch)
                sacred_run.log_scalar('loss', loss.item(), n_batch) #TODO untested

                filename = checkpoint_path+"/GRU_D_epoch_"+str(epoch)+"_step_"+str(n_batch)+".pth"
                print('trying to save file {}'.format(filename) )
                torch.save(model.state_dict(),filename)

                AU_PRCs.append(va_prc)
                train_loss = np.mean(losses)
                # print("Epoch: {} Train: {:.4f}/{:.2f}%, Dev: {:.4f}/{:.2f}%, Test: {:.4f}/{:.2f}% AUC: {:.4f}".format(
                #     epoch, train_loss, train_acc*100, dev_loss, dev_acc*100, test_loss, test_acc*100, auc_score))
                print("Epoch: {} Train loss: {:.4f}, Dev loss: {:.4f}, Val PRC: {:.4f}".format(
                    epoch, train_loss, dev_loss, va_prc))

        train_acc = torch.mean(torch.cat(acc).float())
        train_loss = np.mean(losses)

        train_pred_out = pred
        train_label_out = label

        # save new params
        new_state_dict= {}
        for key in model.state_dict():
            new_state_dict[key] = model.state_dict()[key].clone()

        # compare params
        for key in old_state_dict:
            if (old_state_dict[key] == new_state_dict[key]).all():
                print('Not updated in {}'.format(key))

        """
        # dev loss
        print("Validate...")
        losses, acc = [], []
        label, pred = np.array([]), np.array([])
        model.eval()
        for dev_data, dev_label, num_obs in dev_dataloader:
            # Squeeze the data [1, 33, 49], [1,5] to [33, 49], [5]
            #dev_data = torch.squeeze(dev_data)
            #dev_label = torch.squeeze(dev_label)
            dev_data, dev_label, = dev_data.to(device), dev_label.to(device)

            optimizer.zero_grad()
            # Forward pass : Compute predicted y by passing train data to the model
            #print(dev_data.size())
            #print(model.w_dg_x.weight)
            #print(model.w_dg_x.bias)
            y_pred = model(dev_data)[:,-1,:] #last output
            #print(y_pred.size())

            # Save predict and label
            #pred.append(y_pred.item())
            #label.append(dev_label.item())

            # Compute loss
            loss = criterion(y_pred, dev_label)
            acc.append(
                torch.eq(
                    (torch.sigmoid(y_pred).data > 0.5).float(),
                    dev_label)
            )
            losses.append(loss.item())

            # Save predict and label
            #print(dev_label.detach().numpy().shape)
            #print(y_pred.detach().numpy().shape)
            label = np.append(label, dev_label.detach().cpu().numpy())
            pred = np.append(pred, y_pred.detach().cpu().numpy())
            #print(label.shape)

        dev_acc = torch.mean(torch.cat(acc).float())
        #print(dev_acc)
        #print(label.shape)
        dev_loss = np.mean(losses)

        dev_pred_out = pred
        dev_label_out = label

        va_auc = roc_auc_score(label, pred)
        va_prc = average_precision_score(label, pred)
        print("validation auc:{}".format(va_auc))
        print("validation prc:{}".format(va_prc))

        sacred_run.log_scalar('va_auc', va_auc, n_batch)
        sacred_run.log_scalar('va_prc', va_prc, n_batch)

        AU_PRCs[epoch] = va_prc
        # print("Epoch: {} Train: {:.4f}/{:.2f}%, Dev: {:.4f}/{:.2f}%, Test: {:.4f}/{:.2f}% AUC: {:.4f}".format(
        #     epoch, train_loss, train_acc*100, dev_loss, dev_acc*100, test_loss, test_acc*100, auc_score))
        print("Epoch: {} Train loss: {:.4f}, Dev loss: {:.4f}, Val PRC: {:.4f}".format(
            epoch, train_loss, dev_loss, va_prc))
        """

        # save the parameters
        train_log = []
        train_log.append(model.state_dict())
        #torch.save(model.state_dict(), './save/grud_mean_grud_para.pt')

        #print(train_log)

    return AU_PRCs

#____________________________________________________________________________________________________________________________________________________

@ex.config
def gru_d_config():
    #data config
    file_path = "../mgp-rnn-datadump_labs_vitals_covs_na_thres_500_min_length_7_max_length_200_horizon_0_split_0.pkl"
    data_sources = ['labs','vitals','covs']

    #model config
    hidden_size = 100
    n_layers = 1

    #training config
    n_epochs = 40
    batch_size = 100
    learning_rate = 0.001
    learning_rate_decay =7
    dropout = 0.1
    l2_penalty = 0.001

def count_parameters(model: torch.nn.Module):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

@ex.main
def run_gru_d(_run, file_path, data_sources, batch_size, n_epochs, hidden_size, n_layers, learning_rate, learning_rate_decay, l2_penalty, dropout):

    #setup data
    data = DataContainer(file_path,data_sources) #problem: need same padding for train and va data
    va_data = data.va_data_gru_d()
    tr_data = data.tr_data_gru_d()
    x_mean = tr_data[:,0,:,:].mean(axis=0).mean(axis=1)

    train_dataset = torch.utils.data.TensorDataset(torch.Tensor(tr_data),torch.Tensor(np.expand_dims(data.labels_tr, axis =1)),torch.Tensor(np.expand_dims(data.num_obs_times_tr-1, axis =1)))
    val_dataset = torch.utils.data.TensorDataset(torch.Tensor(va_data), torch.Tensor(np.expand_dims(data.labels_va, axis =1)),torch.Tensor(np.expand_dims(data.num_obs_times_va-1, axis =1)))
    x_mean = torch.Tensor(x_mean)

    #needs int() bc hyperparameter script passes np.int64 and pytorch doesnt like that
    #TODO add num_obs to dataset, then dataloader

    train_dataloader = torch.utils.data.DataLoader(train_dataset, batch_size=int(batch_size), shuffle=True)
    val_dataloader = torch.utils.data.DataLoader(val_dataset, batch_size= int(batch_size), shuffle=True)

    print(va_data.shape)
    print(tr_data.shape)
    print('Data loaded!')
    #print(tr_data[1])

    print("Build model with params:")
    print("hidden_size: {}, dropout: {}, num_layers: {}".format(hidden_size,dropout,n_layers))

    #fit the model
    input_size = 44 # num of variables
    #hidden_size = 100 # same as inputsize
    output_size = 1
    #num_layers = 3 # num of GRU layers (only first layer is GRU-D)
    seq_len = 1264 # max seq len based on data
    model = grud_model(input_size = input_size, hidden_size= int(hidden_size), output_size=output_size, dropout=dropout, dropout_type='mloss', x_mean=x_mean, num_layers=int(n_layers))

    use_cuda = torch.cuda.is_available()
    if use_cuda :
        torch.backends.cudnn.benchmark = True
        model = model.cuda()
        print("Running on GPU")

    criterion = torch.nn.BCELoss()
    #untested
    #logits=preds, targets=minibatch.O_dupe_onehot, pos_weight=self.class_imb)
    #Get class imbalance (for weighted loss):
    case_prev = data.labels_tr.sum()/float(len(data.labels_tr)) #get prevalence of cases in train dataset
    class_imb = torch.tensor(1/case_prev) #class imbalance to use as class weight if losstype='weighted'
    #TODO then we do not need to apply sigmoid to the model output in the model
    criterion = torch.nn.BCEWithLogitsLoss(pos_weight=class_imb)

    if len(_run.observers) > 0:
         checkpoint_path = os.path.join(_run.observers[0].dir, 'model_checkpoints')
    else:
        checkpoint_path = 'model_checkpoints'

    if not os.path.exists(checkpoint_path):
        os.makedirs(checkpoint_path)
    print(model)

    print('Begin training!')
    AU_PRCs = fit(model, criterion, l2_penalty, learning_rate,\
                   train_dataloader, val_dataloader, val_dataloader,\
                   learning_rate_decay, n_epochs, checkpoint_path, _run)

    print(AU_PRCs)
    best_val = np.array(AU_PRCs).max()

    return {'Best Validation AUPRC': best_val}

if __name__ == '__main__':
    ex.run_commandline()


__author__ = "Christian Lyra"
__version__ = "0.1"
__svnversion__ = "$Revision$"
__date__ = "$Date$"
__copyright__ = "Copyright (c) 2007 Christian Lyra"
__license__ = "GPL"
__doc__ = """A plugin to list reminders, but can be used to
show the output of a user command.

To activate, put the following lines in local_conf.py:
plugin.activate("reminders", level=45)
REMINDERS = [ ("cmd", "name", <wrap 0|N>, "string") ]
wrap should be the maximum number of columns, and string if defined would be used to
indent the output. ("/usr/bin/remind -h", "Today", 47, "Reminders for") should output something like:
Reminders for Saturday, 26th May, 2007 (today):
Uncle Bob birthday
"""

#python modules
import os, time, stat, re, copy

#freevo modules
import config, menu, rc, plugin, skin, osd, util
from item import Item

#get the singletons so we get skin info and access the osd
skin = skin.get_singleton()
osd  = osd.get_singleton()

skin.register('reminder', ('screen', 'title', 'info', 'plugin'))

class PluginInterface(plugin.MainMenuPlugin):
    """
    A plugin to list reminders, but can be used to
    show the output of a user command.

    To activate, put the following lines in local_conf.py:
    | plugin.activate('reminders', level=45)
    | REMINDERS = [ ("cmd", "name", <wrap 0|N>, "string") ]

    wrap should be the maximum number of columns, and string if defined would be used to
    indent the output. ("/usr/bin/remind -h", "Today", 47, "Reminders for") should output something like::

        Reminders for Saturday, 26th May, 2007 (today):
            Uncle Bob birthday
    """

    def __init__(self):
        plugin.MainMenuPlugin.__init__(self)

    def items(self, parent):
        return [ RemindMainMenuItem(parent) ]

class RemindItem(Item):
    """
    Item for the menu for one Reminder Type
    """
    def __init__(self, parent):
        self.cmd = None
        self.name = None
        self.wrap = None
        Item.__init__(self, parent)

    def actions(self):
        """
        return a list of actions for this item
        """
        items = [ ( self.remindlines, _('Show Reminders') ) ]
        return items

    def remindlines(self, arg=None, menuw=None):
        lines = []
        for f in self.run_remind(self.cmd, self.wrap):
            mi = menu.MenuItem('%s' % f)
            mi.arg = (mi, menuw)
            lines.append(mi)

        if (len(lines) == 0):
            lines += [menu.MenuItem(_('No Reminders Found'), menuw.back_one_menu, 0)]

        lines_menu = menu.Menu(_('Reminders'), lines)
        menuw.pushmenu(lines_menu)
        menuw.refresh()

    def run_remind(self, cmd, wrap=47):
        """execute the remind command and pretify the output"""

        output = []

        try:
            inst = os.popen(self.cmd)
            f = inst.readlines()
            inst.close()
        except:
            pass
        if int(wrap) > 1:
            for line in f:
                if line !='\n':
                    if self.str and line.rfind(self.str) == 0:
                        pad = ''
                    else:
                        pad = '   '
                    for tmp in self.wrapper(line, int(wrap)).rstrip('\n').split('\n'):
                        output += [ pad + tmp ]
        else:
            output = f
        return output

    def wrapper(self, text, width):
        """
        A word-wrap function that preserves existing line breaks
        and most spaces in the text. Expects that existing line
        breaks are posix newlines.

        from U{http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/148061}
        """
        return reduce(lambda line, word, width=width: '%s%s%s' %
            (line,
            ' \n'[(len(line)-line.rfind('\n')-1
                 + len(word.split('\n',1)[0]) >= width)], word),
            text.split(' '))

class RemindMainMenuItem(Item):
    """
    this is the item for the main menu and creates the list
    of Reminders in a submenu.
    """
    def __init__(self, parent):
        Item.__init__(self, parent, skin_type='reminder')
        self.name = _('Reminders')
        self.reminders = config.REMINDERS

    def config(self):
        return [
            ( 'REMINDERS', None, 'list of tuples containing (command, group, width, header prefix)' )
        ]

    def actions(self):
        """
        return a list of actions for this item
        """
        items = [ ( self.create_reminderstype_menu, _('Reminders types' )) ]
        return items

    def create_reminderstype_menu(self, arg=None, menuw=None):
        remind_types = []

        for (cmd, name, wrap, string) in self.reminders:
            remind_type_item = RemindItem(self)
            remind_type_item.name = name
            remind_type_item.str = string
            remind_type_item.cmd = cmd
            remind_type_item.wrap = wrap
            remind_types += [ remind_type_item ]

        remind_menu = menu.Menu(_('Remind type'), remind_types)
        menuw.pushmenu(remind_menu)
        menuw.refresh()


import json
import os

from pathlib import Path
from jsonschema import RefResolver, Draft7Validator

from itertools import groupby
from copy import deepcopy

from aqt import mw
from aqt.qt import QWidget, QLabel, Qt
from aqt.utils import showInfo

from .util import mapTruthValueToIcon

from .sr_setting_add_replace import SRSettingAddReplace
from .sr_iteration_frontback import SRIterationFrontback

from ..sr_iteration_tab_ui import Ui_SRIterationTab

from ...lib.config import deserialize_iteration, serialize_iteration

def mapSyntaxToExample(openDelim, closeDelim, fieldSeparator, enabled):
    return (f'{openDelim} item1 {fieldSeparator} item2 {closeDelim}' if enabled else '---')

def toMatrix(lst, n):
    return [lst[i:i+n] for i in range(0, len(lst), n)]

# sorts and groups -1 with +1, -2 with +2, etc.
def groupIterations(its):
    return list(filter(lambda v: len(v) == 2, toMatrix(its, 2)))

class SRIterationTab(QWidget):
    def __init__(self):
        super().__init__()

        self.ui = Ui_SRIterationTab()
        self.ui.setupUi(self)

        self.ui.duplicatePushButton.clicked.connect(self.duplicateIteration)
        self.ui.deletePushButton.clicked.connect(self.deleteIteration)
        self.ui.downPushButton.clicked.connect(self.moveDown)
        self.ui.upPushButton.clicked.connect(self.moveUp)
        self.ui.importButton.clicked.connect(self.importDialog)

        self.ui.iterationsTable.currentCellChanged.connect(self.updateButtonsForCurrentCell)
        self.ui.iterationsTable.cellDoubleClicked.connect(self.editIteration)
        self.ui.iterationsTable.setColumnWidth(1, 80)
        self.ui.iterationsTable.setColumnWidth(3, 80)

    def setupUi(self, iterations):
        self.ic = iterationCouples = list(filter(lambda v: len(v) == 2, groupIterations(iterations)))
        self.drawIterations()

        self.updateButtons(False)

    def drawIterations(self):
        self.ui.iterationsTable.clearContents()
        self.ui.iterationsTable.setRowCount(len(self.ic))

        headerLabels = []

        for idx, iter in enumerate(self.ic):
            headerLabels.append(f'Iteration {idx + 1}')

            self.setRowMod(
                idx,
                iter[0].name[1:],
                mapTruthValueToIcon(iter[0].enabled),
                mapSyntaxToExample(iter[0].input_syntax.open_delim,
                                   iter[0].input_syntax.close_delim,
                                   iter[0].input_syntax.field_separator,
                                   iter[0].enabled),
                mapTruthValueToIcon(iter[1].enabled),
                mapSyntaxToExample(iter[1].input_syntax.open_delim,
                                   iter[1].input_syntax.close_delim,
                                   iter[1].input_syntax.field_separator,
                                   iter[1].enabled),
            )

        self.ui.iterationsTable.setVerticalHeaderLabels(headerLabels)

    def setRowMod(self, row, name, frontEnabled, frontText, backEnabled, backText):
        for i, text in enumerate([name, frontEnabled, frontText, backEnabled, backText]):
            label = QLabel()
            label.setText(text)
            label.setAlignment(Qt.AlignCenter)

            self.ui.iterationsTable.setCellWidget(row, i, label)

    def editIteration(self, row, column):
        def saveIterations(newIterations):
            self.ic[row] = newIterations
            self.drawIterations()

        a = SRIterationFrontback(mw, saveIterations)
        a.setupUi(self.ic[row])
        a.exec_()

    #################

    def updateButtonsForCurrentCell(self, currentRow, currentColumn, previousRow, previousColumn):
        self.updateButtons(currentRow != -1)

    def updateButtons(self, state=True):
        self.ui.duplicatePushButton.setEnabled(state)
        self.ui.deletePushButton.setEnabled(state)
        self.ui.downPushButton.setEnabled(state)
        self.ui.upPushButton.setEnabled(state)
        self.ui.importButton.setEnabled(state)

    def duplicateIteration(self):
        newIteration = deepcopy(self.ic[self.ui.iterationsTable.currentRow()])

        self.ic.append([
            newIteration[0]._replace(name=f'{newIteration[0].name}_copy'),
            newIteration[1]._replace(name=f'{newIteration[1].name}_copy'),
        ])
        self.drawIterations()

    def deleteIteration(self):
        if len(self.ic) > 1:
            del self.ic[self.ui.iterationsTable.currentRow()]
            self.drawIterations()

    def moveDown(self):
        i = self.ui.iterationsTable.currentRow()

        if len(self.ic) != 1 and i < len(self.ic) - 1:
            self.ic[i], self.ic[i + 1] = self.ic[i + 1], self.ic[i]
            self.drawIterations()
            self.ui.iterationsTable.setCurrentCell(i + 1, 0)

    def moveUp(self):
        i = self.ui.iterationsTable.currentRow()

        if len(self.ic) != 1 and i > 0:
            self.ic[i], self.ic[i - 1] = self.ic[i - 1], self.ic[i]
            self.drawIterations()
            self.ui.iterationsTable.setCurrentCell(i - 1, 0)

    ###########

    def exportData(self):
        result = []

        # flatten iterations
        for idx, sublist in enumerate(self.ic, start=1):
            result.append(sublist[0])
            result.append(sublist[1])

        return result

    def importDialog(self):
        def addAfterImport(iterations_new):
            self.setupUi(self.ic + [deserialize_iteration(iter) for iter in iterations_new])

        def replaceAfterImport(iterations_new):
            self.setupUi([deserialize_iteration(iter) for iter in iterations_new])

        dirpath = Path(f'{os.path.dirname(os.path.realpath(__file__))}', '../../json_schemas/iterations.json')
        schema_path = dirpath.absolute().as_uri()

        with dirpath.open('r') as jsonfile:
            schema = json.load(jsonfile)
            resolver = RefResolver(
                schema_path,
                schema,
            )

            validator = Draft7Validator(schema, resolver=resolver, format_checker=None)

            dial = SRSettingAddReplace(mw)
            dial.setupUi(
                json.dumps([serialize_iteration(iter) for iterCouple in self.ic for iter in iterCouple], sort_keys=True, indent=4),
                validator,
                addAfterImport,
                replaceAfterImport,
            )
            dial.exec_()


import tornado.ioloop
import tornado.web
from config import config
import os
import dao
from handlers import PackageHandler, PersonHandler, VehicleStationHandler

def make_app(address_dao, package_dao, vehicle_dao, station_dao):

    settings = {
        "static_path": os.path.join(os.path.dirname(__file__), os.pardir, "web"),
        "cookie_secret": "__TODO:_GENERATE2_YOUR_OWN_RANDOM_VALUE_HERE__",
        "xsrf_cookies": True,
    }
    print(settings['static_path'])
    return tornado.web.Application([
        (r"/packages", PackageHandler, dict(package_dao=package_dao,
                                           address_dao=address_dao)),
        (r"/vehicles", VehicleStationHandler, dict(vehicle_dao=vehicle_dao,
                                                station_dao=station_dao)),

       (r'/(.*)', tornado.web.StaticFileHandler, {'path': settings['static_path']}),
    ], **settings)

if __name__ == "__main__":
    #start the server
    packagedao = dao.Packagedao(**config)
    addressdao = dao.Addressdao(**config)
    vehicledao = dao.Vehicledao(**config)
    stationdao = dao.Stationdao(**config)

    app = make_app(addressdao, packagedao, vehicledao, stationdao)
    app.listen(8080)
    tornado.ioloop.IOLoop.current().start()


#Robert Yankou quiz uncorrected
#Warning: do not run this program; it is in an eternal while loop
#All of the mistakes are at the bottom, the critical one being that t does not increase inside the loop

import pygame, time, math
from pygame.locals import *

def slope_x(vx):
    sx = vx
    return sx

def slope_y(vy):
    sy = vy
    return sy

def slope_z(vz):
    sz = vz
    return sz

def slope_vx(vy,vz,By,Bz):
    svx = vy*Bz-vz*By
    return svx

def slope_vy(vx,vz,Bx,Bz):
    svy = vz*Bx-vx*Bz
    return svy

def slope_vz(vx,vy,Bx,By):
    svz = vx*By-vy*Bx
    return svz

Bx = 0
By = 0
Bz = 10
x = 0
y = 0
z = 0
vx = 1
vy = 1
vz = 1
t = 0
tf = 10
h = 0.05

running = True
W,H = 500,500
pygame.init()
screen = pygame.display.set_mode((W,H))
pygame.draw.line(screen,(128,128,128),(0,H/2),(W,H/2),1)
pygame.draw.line(screen,(128,128,128),(W/2,0),(W/2,H),1)
pygame.display.flip()

xold = W/2 + x
yold = H/2 - y
pygame.draw.circle(screen,(255,0,0),(xold,yold),5,1)

print(x,y,z,vx,vy,vz)
print("KE= ", ((vx**2+vy**2+vz**2)/2))

while abs(t)<tf:
      M1=slope_x(vx)
      J1=slope_vx(vy,vz,By,Bz)
      N1=slope_y(vy)
      K1=slope_vy(vx,vz,Bx,Bz)
      O1=slope_z(vz)
      L1=slope_vz(vx,vy,Bx,By)
      M2=slope_x(vx+J1*h/2)
      J2=slope_vx(vy+K1*h/2,vz+L1*h/2,By,Bz)
      N2=slope_y(vy+K1*h/2)
      K2=slope_vy(vx+J1*h/2,vz+L1*h/2,Bx,Bz)
      O2=slope_z(vz+L1*h/2)
      L2=slope_vz(vx+J1*h/2,vy+K1*h/2,Bx,By)
      M3=slope_x(vx+J2*h/2)
      J3=slope_vx(vy+K2*h/2,vz+L2*h/2,By,Bz)
      N3=slope_y(vy+K2*h/2)
      K3=slope_vy(vx+J2*h/2,vz+L2*h/2,Bx,Bz)
      O3=slope_z(vz+L2*h/2)
      L3=slope_vz(vx+J2*h/2,vy+K2*h/2,Bx,By)
      M4=slope_x(vx+J3*h)
      J4=slope_vx(vy+K3*h,vz+L3*h,By,Bz)
      N4=slope_y(vy+K3*h)
      K4=slope_vy(vx+J3*h,vz+L3*h,Bx,Bz)
      O4=slope_z(vz+L3*h)
      L4=slope_vz(vx+J3*h,vy+K3*h,Bx,By)

      x += (M1+2*M2+2*M3+M4)*h/6
      y += (N1+2*N2+2*N3+N4)*h/6
      z += (O1+2*O2+2*O3+O4)*h/6
      vx += (J1+2*J2+2*J3+J4)*h/6
      vy += (K1+2*K2+2*K3+K4)*h/6
      vz += (L1+2*L2+2*L3+L4)*h/6

      xnew = W/2 + x
      ynew = H/2 - y

      pygame.draw.aaline(screen,(0,0,255),(xold,yold),(xnew,ynew),1)
      pygame.display.flip()

      xold = xnew
      yold = ynew

print(x,y,z,vx,vy,vz)
print("KE= ", ((vx**2+vy**2+vz**2)/2))

while running:
      for event in pygame.event.get()==Quit:
          running = False
pygame.display.quit()

#Left m&q out completely
#Assumed values for vx,vy,vz,Bz,tf&h
#Did not fit or scale graph, assumed values for W&H
#Could have used 2 functions instead of 6
#Did not have to set B's as inputs of functions since they are constant


# -*- coding: UTF-8 -*-
from django.contrib import admin
from catalogs.models import CatalogItem

__author__ = 'Razzhivin Alexander'
__email__ = 'admin@httpbots.com'

class CatalogItemAdmin(admin.ModelAdmin):
    def save_model(self, request, obj, form, change):
        obj.save()
        obj.object_id=obj.id
        obj.save()

admin.site.register(CatalogItem, CatalogItemAdmin)


from .oauth2 import OAuth2 as FalconAuth

# pylint: disable=R0902  # Eight is reasonable here

class ServiceClass:
    """
    Base class of all service classes. Contains the default __init__ method.
    """
    def __init__(self: object, auth_object: object = None,
                 creds: dict = None, base_url: str = "https://api.crowdstrike.com",
                 proxy: dict = None, **kwargs) -> object:
        """
        Instantiates the base class, ingests the authorization token,
        and initializes the headers and base_url global variables.
        """
        access_token, self.ssl_verify, self.timeout = self.parse_keywords(kwargs)
        self.refreshable = False

        if auth_object:
            self.auth_object = auth_object
            if not self.authenticated():
                _ = self.auth_object.token()
                if _["status_code"] == 201:
                    self.token = _["body"]["access_token"]
                    self.headers = {'Authorization': 'Bearer {}'.format(self.token)}
                else:
                    self.token = False
                    self.headers = {}
            else:
                self.token = self.auth_object.token_value
                self.headers = {'Authorization': 'Bearer {}'.format(self.token)}

            self.base_url = auth_object.base_url
            self.ssl_verify = auth_object.ssl_verify
            self.proxy = auth_object.proxy
            # At this point in time, you cannot override the auth_object's timeout per class
            self.timeout = auth_object.timeout
            self.refreshable = True
        else:
            if creds:
                auth_object = FalconAuth(creds=creds)
                self.auth_object = auth_object
                _ = self.auth_object.token()
                if _["status_code"] == 201:
                    self.token = _["body"]["access_token"]
                    self.headers = {'Authorization': 'Bearer {}'.format(self.token)}
                else:
                    self.token = False
                    self.headers = {}
                self.refreshable = True
            else:
                self.auth_object = None
                self.headers = {'Authorization': 'Bearer {}'.format(access_token)}

            self.base_url = base_url
            self.proxy = proxy

    def authenticated(self):
        """
        Authenticates using the credentials provided.
        """
        result = None
        if self.auth_object:
            result = self.auth_object.authenticated()

        return result

    def token_expired(self):
        """
        Returns a boolean reflecting token expiration status
        """
        result = None
        if self.auth_object:
            result = self.auth_object.token_expired()

        return result

    @staticmethod
    def parse_keywords(passed_keywords: dict):
        """
        Parses passed keywords to _init, setting defaults
        """
        access_token = None
        ssl_verify = True
        timeout = None
        for key, val in passed_keywords.items():
            if key.lower() == "access_token":
                access_token = val
            if key.lower() == "ssl_verify":
                ssl_verify = val
            if key.lower() == "timeout":
                timeout = val

        return access_token, ssl_verify, timeout


from django.shortcuts import render, redirect
from django.contrib.auth import login, authenticate
from django.contrib.auth.models import Group
from django.contrib import messages
from django.contrib.auth.forms import UserCreationForm
from django.views.generic.base import TemplateView
from django.views.generic import ListView, CreateView
from .forms import UserRegisterForm
from django.core.paginator import Paginator
from .models import User

def register(request):
    form = UserRegisterForm()
    if request.method == "POST":
        form = UserRegisterForm(request.POST)
        if form.is_valid():
            u = form.save()
            try:
                group = Group.objects.get(name="default_user")
                group.user_set.add(u)
            except Exception as er:
                # raise
                print(er)
            print("OK")
            authenticate(u)
            message = "Successfully !"
            messages.add_message(request, messages.SUCCESS, message)
            return redirect("/")
        else:
            message = "Error !"
            messages.add_message(request, messages.ERROR, message)
            return render(request, "auth/register.html", {"form": form})

    return render(request, "auth/register.html", {"form": form})

def employee(request):
    user_list = User.objects.all()
    paginator = Paginator(user_list, 2)  # Show 25 contacts per page.
    page_number = request.GET.get("page")
    object_list = User.objects.filter(is_staff=True).page_obj = paginator.get_page(page_number)

    context = {
        'object_list':object_list,

    }

    return render(request, 'contragents.html', context)

class KontragentFormView(CreateView):
    model = User
    template_name = 'forms/contragent_form.html'
    fields = ['password', 'is_superuser', 'username', 'first_name', 'last_name', 'is_staff', 'is_active', 'email', 'status', 'birthday', 'age', 'phone', 'address']
    success_url = '/'


from setuptools import setup, find_packages
import sys, os

version = '0.1'

#packages = find_packages("dab",exclude=["*.tests.*", "tests.*", "tests"])

# setup(
#     name="deepin-auto-build",
#     author="Heysion Yuan",
#     packages = packages,
#     package_dir = {"":"dab"},
# )

setup(name='deepin-auto-build',
      version=version,
      description="deepin auto build  platform",
      long_description="""build package & customer discs , manager repo
""",
      classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
      keywords='deepin-auto-build deepin-maker-discs dab dmd',
      author='heysion',
      author_email='heysion@deepin.com',
      url='deepin.org',
      license='GPLv3',
      packages=find_packages(exclude=['ez_setup', 'examples', 'tests']),
      include_package_data=True,
      zip_safe=False,
      install_requires=[
          # -*- Extra requirements: -*-
      ],
      entry_points="""
      # -*- Entry points: -*-
      """,
      )


from django.urls import path
from . import views

urlpatterns = [
    path('login/', views.login, name='login'),
    path('logout/', views.logout, name='logout'),
    path('home/', views.home, name='home'),
    path('open_camera/', views.open_camera, name='open_camera'),
    path('t_history/', views.t_history, name='t_history'),
    path('s_history/', views.s_history, name='s_history'),
    path('attendance_record/', views.attendance_record, name='attendance_record'),
    path('exist_photo/', views.exist_photo, name='exist_photo'),
    path('take_photo/', views.take_photo, name='take_photo'),
]


"""
This file is straight-forward, it simply runs the experiments and saves the data.
Experiments are run (and separate data files are saved) for all combinations of the
following parameters:
    (1) Topology
    (2) Algorithm solution
    (3) Opinion distribution function (uniform/polarized)
    (4) Community-aware or Community-agnostic?
"""

import itertools
import json
import matplotlib.pyplot as plt
import seaborn as sns
import pandas as pd
import os
import random
import Simulation
import time
import networkx as nx

from Algorithms.Heuristics import *
from Algorithms.LAIM       import *
from Algorithms.Proposed   import *
from Algorithms.TIM_Plus   import TIM_plus_solution
from Diffusion.Model       import *
from scipy.stats           import arcsine
from tqdm                  import tqdm

sns.set_style("darkgrid")
RESULTS_PATH = os.path.join("..", "out", "results")
META_PATH = os.path.join(RESULTS_PATH, "meta.json")

uniform   = lambda: random.random(); uniform.__name__ = "uniform"
polarized = lambda: arcsine.rvs();   polarized.__name__ = "polarized"

# Dependent variables.
default_topologies = ["amazon", "dblp", "eu-core", "facebook", "twitter"]
default_algorithms = [
    TIM_plus_solution, LAIM_solution, fast_LAIM_solution, # Baselines
    opinion_degree_solution,                              # Proposed
    degree_solution, IRIE_solution, min_opinion_solution  # Heuristics
]
seed_sizes = list(range(0, 50+1, 10)); seed_sizes[0] = 1
# opinion_distrs = [uniform, polarized]
# use_communities = [False, True]
opinion_distrs = [polarized, uniform]
use_communities = [True, False]
alg_codes = {
    "TIM+": TIM_plus_solution,
    "LAIM": LAIM_solution,
    "fast_LAIM": fast_LAIM_solution,
    "new": new_solution,
    "opinion_degree": opinion_degree_solution,
    "degree": degree_solution,
    "IRIE": IRIE_solution,
    "min_opinion": min_opinion_solution
}

# Constant parameters.
n_trials = 100
t_horizon = 100
random_seed = 16785407 # a large *prime* number

if __name__ == "__main__":
    import argparse

    ## Use the argparse class to extract passed-in arguments from the command line. This
    ## is done to simplify the process of running multiple algorithms in parallel without
    ## explicit multi-threading.
    parser = argparse.ArgumentParser()
    parser.add_argument("--alg")
    parser.add_argument("--topo")
    args = parser.parse_args()

    if args.alg is None:
        algorithms = default_algorithms
    else:
        alg_func = alg_codes[args.alg]
        algorithms = [alg_func]
    if args.topo is None:
        topologies = default_topologies
    else:
        if args.topo not in Simulation.TOPO_CODES:
            raise ValueError("Invalid value for arg `topo`.")
        topologies = [args.topo]

    ## Run the experiments over all combinations of the paramters.
    param_combinations = itertools.product(topologies, algorithms, opinion_distrs, use_communities)
    for (topo_code, alg, opinion_distr, use_comm) in param_combinations:
        filename = "topo={}_alg={}_comm={}_{}".format(
            topo_code, alg.__name__.replace("_solution", ""), use_comm,
            opinion_distr.__name__
        )
        results = Simulation.run(
            topo_code          = topo_code,
            algorithm          = alg,
            seed_sizes         = seed_sizes,
            time_horizon       = t_horizon,
            n_trials           = n_trials,
            opinion_distr_func = opinion_distr,
            random_seed        = random_seed,
            use_communities    = use_comm,
            pbar_desc          = filename
        )
        results.to_csv(os.path.join(RESULTS_PATH, filename + ".csv"))


from django.shortcuts import render,HttpResponse, redirect
from django.contrib.auth.decorators import login_required

from . import parse_lotto
import json
import random

from accounts.forms import CustomUserCreationForm, CustomAuthenticationForm
from accounts.models import CustomUser

# Create your views here.

def index(request):
    return render(request, 'index.html')

@login_required
def lotto(request):
    if request.user.money < 1000:
        return redirect('/lotto/#GasStation')

    user = CustomUser.objects.get(id=request.user.id)
    user.money -= 1000
    user.save()

    context = {
        'numbers' : range(1,46)
    }

    return render(request, 'lotto.html', context)

@login_required
def request_lotto_number_sum(request):
    lotto_data = parse_lotto.get_all_lotto_data()
    sum_lotto_data = parse_lotto.get_lotto_number_sum(lotto_data)

    context = {
        'sum_lotto_data' : sum_lotto_data
    }

    return HttpResponse(json.dumps(context), content_type='application/json')

@login_required
def request_lotto_number_each_win_count(request):
    lotto_data = parse_lotto.get_all_lotto_data()
    lotto_number_each_win_count = parse_lotto.get_lotto_number_each_win_count(lotto_data)

    context = {
        'lotto_number_each_win_count' : lotto_number_each_win_count
    }

    return HttpResponse(json.dumps(context), content_type='application/json')

@login_required
def request_lotto_number_duration_win_count(request):
    lotto_data = parse_lotto.get_all_lotto_data()
    lotto_number_duration_win_count = parse_lotto.get_lotto_number_duration_win_count(lotto_data)

    context = {
        'lotto_number_duration_win_count' : lotto_number_duration_win_count
    }

    return HttpResponse(json.dumps(context), content_type='application/json')

@login_required
def request_lotto_number_year_win_count(request):
    target_year = int(request.GET.get('year'))
    target_number = int(request.GET.get('number'))

    if target_year != None and target_number != None:
        lotto_data = parse_lotto.get_all_lotto_data()
        lotto_number_year_win_count = parse_lotto.get_lotto_number_year_win_count(lotto_data, target_number, target_year)

        context = {
            'lotto_number_year_win_count' : lotto_number_year_win_count
        }

        return HttpResponse(json.dumps(context), content_type='application/json')

    return HttpResponse(status=403)

@login_required
def request_lotto_number_expectation(request):
    expectation = random.sample(range(1,46), 7)

    context = {
        'expectation' : expectation
    }

    return HttpResponse(json.dumps(context), content_type='application/json')

# lotto_data_test
#       This function is for testing lotto_data

def lotto_data_test(request):
    lotto_data = parse_lotto.get_all_lotto_data()

    lotto_number_sum = parse_lotto.get_lotto_number_sum(lotto_data)

    result =[]

    for tmp in range(1,46):
        a = lotto_number_sum[tmp]

        one = {
            'name':tmp,
            'y':a,
            'drilldown':tmp
        }

        result.append(one)

    context={
        'result': result
    }

    return HttpResponse(json.dumps(context),content_type="application/json")

def lotto_data_highchaet(request):
    return render(request,'test/highchart.html')

def lotto_data_test2(request):
    lotto_data = parse_lotto.get_all_lotto_data()

    lotto_number_sum = parse_lotto.get_lotto_number_sum(lotto_data)
    lotto_number_win_count = parse_lotto.get_lotto_number_win_count(lotto_data)
    lotto_number_each_win_count = parse_lotto.get_lotto_number_each_win_count(lotto_data)

    context = {
        'lotto_data' : lotto_data,
        'lotto_number_sum' : lotto_number_sum,
        'lotto_number_win_count' : lotto_number_win_count,
        'lotto_number_each_win_count' : lotto_number_each_win_count
    }

    return render(request, 'test/lotto_data.html', context)


import pandas as pd

import matplotlib.pyplot as plt

DATE_COLUMNS = ['start_timestamp', 'task_start_label',
                'suite_start', 'suite_end', 'task_end_label', 'end_timestamp']

def simple_display(df, column):
    """Straightforward box plot."""
    show_start_delay = pd.DataFrame({col: vals[column]
                                     for col, vals in df.groupby('task_name_nochunks')})
    meds = show_start_delay.median().sort_values()
    meds = meds.tail(30)  # Last 30 values, or the graph is unreadable
    show_start_delay = pd.DataFrame(
        {col: vals[column] for col, vals in df.groupby('task_name_nochunks') if col in meds.index})
    # Recalculate for shorter list, as we need the index
    meds = show_start_delay.median().sort_values()
    show_sorted = show_start_delay[meds.index]
    show_sorted.boxplot(vert=False, sym='')
    plt.show()

def display_with_task_count(df, column):
    """Add a second y axis showing task count."""
    fig, ax = plt.subplots()
    ax.set_ylabel('SUITE-START delay:Total Duration ratio')
    ax3 = ax.twinx()
    ax3.set_frame_on(True)
    ax3.patch.set_visible(False)
    ax3.set_ylabel('Task Count')
    show_start_delay = pd.DataFrame({col: vals[column]
                                     for col, vals in df.groupby('task_name_nochunks')})
    meds = show_start_delay.median().sort_values()
    meds = meds.tail(30)  # Last 30 values, or the graph is unreadable
    show_start_delay = pd.DataFrame(
        {col: vals[column] for col, vals in df.groupby('task_name_nochunks') if col in meds.index})
    # Recalculate for shorter list, as we need the index
    meds = show_start_delay.median().sort_values()
    show_sorted = show_start_delay[meds.index]
    show_sorted.boxplot(ax=ax, sym='', rot=90)
    task_counts = pd.DataFrame([row for _, row in df['task_name_nochunks'].value_counts(
    ).reset_index().iterrows() if row['index'] in meds.index])
    task_counts.set_index('index', inplace=True)
    tc = pd.DataFrame([task_counts.loc[col, 'task_name_nochunks'] for col in show_sorted.columns])
    tc.plot.line(ax=ax3, legend=False)
    plt.show()

def main():
    """Main."""
    df = pd.read_csv('autoland_test_logfiles.csv',
                     infer_datetime_format=True, parse_dates=DATE_COLUMNS)

    # Create new columns
    df['start_label_delay'] = df['task_start_label'] - df['start_timestamp']
    df['suite_start_delay'] = df['suite_start'] - df['task_start_label']

    # Graphs have difficulty with timedeltas
    df['start_label_delay_s'] = df['start_label_delay'] / pd.Timedelta(seconds=1)
    df['suite_start_delay_s'] = df['suite_start_delay'] / pd.Timedelta(seconds=1)

    # Remove chunking to aggregate tasks a bit.
    df['task_name_nochunks'] = df['task_name'].apply(lambda x: x.rstrip('1234567890-'))
    df['total_duration'] = df['end_timestamp'] - df['task_start_label']
    df['total_duration_s'] = df['total_duration'] / pd.Timedelta(seconds=1)
    df['suite_delay_ratio'] = df['suite_start_delay_s'] / df['total_duration_s']
    df['suite_delay_percentage'] = df['suite_delay_ratio'] * 100

    # simple_display(df, 'start_label_delay_s')
    # simple_display(df, 'suite_start_delay_s')

    # display_with_task_count(df, 'start_label_delay_s')
    display_with_task_count(df, 'suite_start_delay_s')

if __name__ == "__main__":
    main()


# test splines
import numpy as np
import pytest
# from os.path import dirname
# sys.path.append(dirname(__file__))
import sys
import os
sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../../")

# from . import splines

# sys.path.append(os.path.dirname(os.path.realpath(__file__)) + "/../")
# import splines
from functions import splines
from functions import splines_rpy2
# from imp import reload
# reload(splines)
# reload(splines_rpy2)

testdata = [(1, 100, 10, 3, True),
            (1, 100, 10, 3, False),
            (1, 100, 10, 2, True),
            (1, 100, 10, 2, False),
            (-1, 100, 5, 2, True),
            (-1, 2, 5, 2, True),
            (-1, 100, 50, 2, True),
            (-1, 100, 50, 1, True),
            (-1, 100, 50, 5, True)]

@pytest.mark.parametrize("start, end, n_bases, spline_order, add_intercept", testdata)
def test_compare_tuple(start, end, n_bases, spline_order, add_intercept):

    X1, S1, knots1 = splines.get_gam_splines(start=start, end=end,
                                             n_bases=n_bases,
                                             spline_order=spline_order,
                                             add_intercept=add_intercept)

    X2, S2, knots2 = splines_rpy2.get_gam_splines(start=start, end=end,
                                                  n_bases=n_bases,
                                                  spline_order=spline_order,
                                                  add_intercept=add_intercept)

    assert np.abs(knots1 - knots2).max() < 1e-5
    assert np.abs(X1 - X2).max() < 1e-5
    assert np.abs(S1 - S2).max() < 1e-5


import bpy
import random
import datetime

from blenderbrain.common import check_regexs
from blenderbrain.common import calculate_age

class kn_help:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"help"])
        if re_test:
            return True
        return False

    def execute(self, info):
        help_text = [
            "Help"
        ]
        return help_text

class kn_who_father:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"who\s+is\s+your\s+(father|dad)"])
        if re_test:
            return True
        return False

    def execute(self, info):
        help_text = [
            "My father is Ton Roosendaal"
        ]
        return help_text

class kn_how_old:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"how\s+old", r"your\s+age"])
        if re_test:
            return True
        return False

    def execute(self, info):
        age = calculate_age(datetime.datetime(1993, 1, 2))
        help_text = [
            "I was born January 2nd 1993, so I'm {} years old".format(age)
        ]
        return help_text

class kn_what_are_doing:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"what\s+are\s+you\s+doing", r"what\s+do\s+you\s+do"])
        if re_test:
            return True
        return False

    def execute(self, info):
        help_text = [
            "I'm refreshing the UI",
            "I'm counting vertices",
            "I'm reading what you write",
            "I'm waiting for your orders",
            "I'm doing some complex math"
        ]
        return [random.choice(help_text)]

class kn_where_are_you:
    def check(self, info):
        regs = [
            r"where\s+are\s+you",
            r"(blender\s+)?location"
        ]
        re_test = check_regexs(info["question"], regs)
        if re_test:
            return True
        return False

    def execute(self, info):
        help_text = [
            "Blender location: {}".format(bpy.app.binary_path)
        ]
        return help_text

class kn_are_male_female:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"are\s+you\s+(a\s+)?(male|boy|female|girl|gal)"])
        if re_test:
            return True
        return False

    def execute(self, info):
        help_text = [
            "I'm not quite sure how to answer that. I'm software",
            "I'm just a bunch of bits.",
            "I'm just ones and zeros",
            "I'm just a software'"
        ]
        return [random.choice(help_text)]

class kn_are_what_name:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"what\s+is\s+your\s+name"])
        if re_test:
            return True
        return False

    def execute(self, info):
        help_text = [
            "My name is Blender"
        ]
        return help_text

class kn_who_are_you:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"(who|what)\s+are\s+you"])
        if re_test:
            return True
        return False

    def execute(self, info):
        help_text = [
            "I'm your friendly free software Blender"
        ]
        return help_text

class kn_version:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"(blender\s+)?version"])
        if re_test:
            return True
        return False

    def execute(self, info):
        data = [
            bpy.app.version_string,
            bpy.app.build_branch.decode("UTF-8"),
            bpy.app.version_cycle,
            bpy.app.build_platform.decode("UTF-8"),
            bpy.app.build_system.decode("UTF-8")
        ]
        help_text = [
            "Blender Version: {} {} {} {} {}".format(*data),
            "  Alembic: {}".format(bpy.app.alembic),
            "  FFMPEG: {}".format(bpy.app.ffmpeg),
            "  OCIO: {}".format(bpy.app.ocio),
            "  OIIO: {}".format(bpy.app.oiio),
            "  OpenVDB: {}".format(bpy.app.openvdb),
            "  SDL: {}".format(bpy.app.sdl),
        ]
        return help_text

class kn_temp:
    def check(self, info):
        re_test = check_regexs(info["question"], [r"temp(oral)?(\s+folder)?"])
        if re_test:
            return True
        return False

    def execute(self, info):
        help_text = [
            "Temp folder: {}".format(bpy.app.tempdir)
        ]
        return help_text

classes = [
    kn_temp,
    kn_version,
    kn_help,
    kn_how_old,
    kn_who_father,
    kn_what_are_doing,
    kn_where_are_you,
    kn_are_male_female,
    kn_are_what_name,
    kn_who_are_you
]


from django.conf.urls import url

from . import views
app_name = 'student'

urlpatterns = [
    url(r'^$', views.index, name='index'),

    url(r'^1/$', views.read, name='read'),
    url(r'^2/$', views.search, name='search'),
]


# -*- coding: utf-8 -*-
# @Author: joaopn
# @Date:   2019-07-19 23:08:49
# @Last Modified by:   joaopn
# @Last Modified time: 2019-07-22 20:10:23

"""
Runs the avalanche analysis for all datasets in a folder, averaging over unique filenames ending with '_rXX.hdf5'.
- Skips averaging for -reps 0.
- Doesn't depend on specific filename syntax.
"""

from analysis import parser, dataset
import argparse

def parametersDefault():

	#default Parameters
	binsizeDefault="1,2,4,8"
	thresholdDefault = 3
	repsDefault = 1
	datatypeDefault = 'coarse'
	datafolderDefault = 'dat/'
	bw_filterDefault = True
	fit_limDefault = 50

	#Parse input
	parser = argparse.ArgumentParser()

	#Add single parameters
	parser.add_argument("-t", "--threshold",
		type=float, nargs='?', const=1, default=thresholdDefault)
	parser.add_argument("--reps",
		type=int,   nargs='?', const=1, default=repsDefault)
	parser.add_argument("--datatype",
		type=str,   nargs='?', const=1, default=datatypeDefault)
	parser.add_argument("--datafolder",
		type=str,   nargs='?', const=1, default=datafolderDefault)
	parser.add_argument("--datamask",
		type=str,   nargs='?', const=1, default=None)
	parser.add_argument("--bw_filter",
		type=bool,  nargs='?', const=1, default=bw_filterDefault)
	parser.add_argument("--fit_lim",
		type=int,   nargs='?', const=1, default=fit_limDefault)

	#Adds string of binsizes
	parser.add_argument("-b","--binsize",type=str,nargs='?',const=1,default=binsizeDefault)
	args = parser.parse_args()
	args.binsize = [int(item) for item in args.binsize.split(',')]

	return args

if __name__ == "__main__":

	#Parse input
	args       = parametersDefault()
	binsize    = args.binsize
	threshold  = args.threshold
	reps       = args.reps
	datatype   = args.datatype
	datafolder = args.datafolder
	datamask   = args.datamask
	bw_filter  = args.bw_filter
	fit_lim = args.fit_lim

	#Gets the unique datasets in the folder
	datasets_unique = parser.sim_find_unique(datafolder)

	#Runs it for each unique dataset
	for dataset in datasets_unique:

		print('Running: '+dataset)

		#Rebuilds dataset list
		filepath = parser.sim_add_reps(datafolder+dataset,reps)

		#Runs analysis
		fig_loc = datafolder + dataset
		dataset.sim_plot_deltaT(filepath=filepath,deltaT=binsize, datatype=datatype,threshold=threshold,S_fit_max=fit_lim,bw_filter=bw_filter,save_fig=fig_loc)