#!/usr/bin/env python3
"""
This script (or maybe scripts eventually?) will be used to transfer
assets and metadata from our local ResourceSpace to Internet Archive.
"""
import csv
from google_drive_downloader import GoogleDriveDownloader # from https://github.com/ndrplz/google-drive-downloader/blob/master/google_drive_downloader/google_drive_downloader.py
import hashlib
from internetarchive import upload
import os.path
import re
import requests
import subprocess
import sys
import time
import urllib

# # COUNTER IS FOR TESTING PURPOSES
# counter=1

rightsStatement = """Copyright ©TVTV. This work is protected by U.S. Copyright Law \
(Title 17, U.S.C.). In addition, its reproduction may be restricted by terms \
of gift or purchase agreements, donor restrictions, privacy and publicity \
rights, licensing and trademarks. This work is made accessible ONLY for purposes of \
education and research. Transmission or reproduction of works protected by \
copyright beyond that allowed by fair use requires the written permission \
of the copyright owners. Works not in the public domain may not be \
commercially exploited without permission of the copyright owner. \
Responsibility for any use rests exclusively with the user.\n
Please send questions and comments to bampfa@berkeley.edu.
"""
# rightsStatement = """This work may be protected by the U.S. Copyright Law \
# (Title 17, U.S.C.). In addition, its reproduction may be restricted by terms \
# of gift or purchase agreements, donor restrictions, privacy and publicity \
# rights, licensing and trademarks. This work is made accessible ONLY for purposes of \
# education and research. Transmission or reproduction of works protected by \
# copyright beyond that allowed by fair use requires the written permission \
# of the copyright owners. Works not in the public domain may not be \
# commercially exploited without permission of the copyright owner. \
# Responsibility for any use rests exclusively with the user.\n
# Berkeley Art Museum and Pacific Film Archive has made efforts in all \
# cases to secure permission to display copyrighted works from rights owners; \
# we are eager to hear from rights owners with any questions or concerns \
# regarding this display.\n
# If you are a legitimate copyright holder to this work and \
# would like to discuss removing it from public display, please send requests \
# to bampfa@berkeley.edu.
# """

class User:
	'''
	Define a user who will be connecting to
	ResourceSpace and Internet Archive.
	'''
	def __init__(self):
		self.define_rs_user()
		self.define_ia_user()

	def define_rs_user(self):
		rsUserName = input("enter resourcespace user name:")
		if not rsUserName.isalnum():
			rsUserName = urllib.parse.quote_plus(rsUserName)
		self.rsUserName = rsUserName
		self.rsAPIkey = input("enter your resourcespace API key:")

	def define_ia_user(self):
		'''
		The Internet Archive Python library works by storing archive.org credentials
		in an initialization file at /home/.config/ia.ini. The script will look
		for that file. If it doesn't exist, it will run the Internet Archive
		CLI "configure" function to set username and password interactively.
		'''
		try:
			os.path.isfile('/home/.config/ia.ini')
		except FileNotFoundError:
			print("Your Internet Archive credentials are missing! There should be a config file at /home/.config/ia.ini.")
			print("If this script does not prompt you to enter your archive.org credentials, please exit and run `ia configure` in the terminal.")
			# not sure that below works in all cases - should run the bashCommand as defined in line below, then communicate output errors
			bashCommand = "ia configure"
			process = subprocess.Popen(bashCommand.split(), stdout=subprocess.PIPE)
			output, error = process.communicate()
		else:
			pass

class ResourceSpaceAPI:
	'''
	Define location of ResourceSpace assets
	and how to query the API. Largely based on
	https://github.com/pixuenan/RS-python-API/blob/master/RSAPI.py
	'''
	def __init__(self,_user=None):
		self.edithServer = "resourcespace.bampfa.berkeley.edu"
		self._user = _user

	def query(self, function_to_query, parameters, _user):
		'''
		Construct an RS API query:
		1. Define the query: combination of username, the ResourceSpace function,
			and the parameters of the query (e.g. ResourceSpace ID)
		2. Hash user's private API key with the query itself to create a unique
			query string ("sign")
		3. Combine the hashed query string with the query itself in a query URL
			(linked to the Edith server)
		For more detail: https://www.resourcespace.com/knowledge-base/api/
		'''
		query = "user={}&function={}&{}".format(
			_user.rsUserName,
			function_to_query,
			parameters
			)
		sign = hashlib.sha256(_user.rsAPIkey.encode()+query.encode())
		sign = sign.hexdigest()
		queryURL = "https://{}/api/?{}&sign={}".format(
			self.edithServer,
			query,
			sign
			)
		# get the result of API query, i.e. what is returned by the query URL
		# print(queryURL)
		result = requests.post(queryURL)
		# try:
		# 	# get the result of API query, i.e. what is returned by the query URL
		# 	result = queryURL
		# except (IOError, UnicodeDecodeError) as err:
		# 	print(err)
		print(result.text)
		httpStatus = result.status_code
		if httpStatus == 200:
			return result.text.replace("\\","").replace("\"","")
		else:
			return None

class Asset:
	'''
	Define an asset, which could be simple (one file)
	or complex (a set of files), plus metadata
	'''
	def __init__(
		self,
		rsAssetID = None,
		# localAssetPaths=None,
		assetMetadata = {},
		_user = None,
		mediaType = None
		):
		self.localAssetPaths = []
		self.assetMetadata = assetMetadata
		self.identifier = None
		self.creator = None
		self.title = None
		self.date = None
		self.subject = None
		self.description = None
		self.notes = None
		self.mediaType = mediaType
		self.rsAssetID = self.assetMetadata['Resource ID(s)'] # this value will come from a metadata CSV file
		self.collection = ['stream_only','pacificfilmarchive'] # collection can be an array
		#self.license = 'https://creativecommons.org/licenses/by-nc-nd/4.0/'
		self._user = _user
		self.rsAPI = ResourceSpaceAPI(_user)

	def get_local_asset_path(self):
		# see https://www.resourcespace.com/knowledge-base/api/get_resource_path
		# construct parameters of API call as a string
		parameters = (
			"param1={}"
			"&param2=1"
			"&param3="
			"&param4="
			"&param5={}".format(
				self.rsAssetID,
				self.mediaType
				)
			)
		# query API for filepath of primary asset as hosted on ResourceSpace - COMMENTED OUT FOR TESTING
		self.primaryAssetPath = self.rsAPI.query(
			"get_resource_path",
			parameters,
			self._user
			)

		### THIS IS FAKE STUFF FOR TESTING. THERE ARE 3 FAKE FILES: 1bampfaTVTV.mp4, 2bampfaTVTV.mp4, 3bampfaTVTV.mp4
		# global counter
		# self.primaryAssetPath = os.path.join("/Users/bampfa/Documents/GitHub/rs2ia/fakes/",str(counter)+"bampfaTVTV.mp4")
		# counter += 1
		# END FAKE STUFF FOR TESTING

		print("PRIMARY ASSET PATH:")
		print(self.primaryAssetPath)
		self.localAssetPaths.append(self.primaryAssetPath)

	def get_local_alternative_asset_paths(self):
		# get filepaths for alternative files associated with the primary asset.
		# see https://www.resourcespace.com/knowledge-base/api/get_alternative_files
		# construct parameters of API call as a string
		parameters = "param1={}&param2=&param3=".format(self.rsAssetID)
		# query API; result is a dictionary of information about
		# the alternative files, but no actual filepaths
		# also, it's not a valid Python dict, it's maybe a PHP array?
		self.alternativeAssetDict = self.rsAPI.query(
			"get_alternative_files",
			parameters,
			self._user
			)
		print("ALT ASSETS FROM RS:")
		print(self.alternativeAssetDict)

		# get the ref ID for each alternative asset and make a new query with that ID
		# there should be a 1:1 relationship between
		# the matched ref #'s and file extensions
		alts = {}
		refNumbers = [ref[1] for ref in re.findall(r"({ref\:)([0-9]+)",self.alternativeAssetDict)]
		extensions = [ext[1] for ext in re.findall(r"(,file_extension:)(\w{0,4})",self.alternativeAssetDict)]
		if not len(refNumbers) == len(extensions):
			print("ALTERNATIVE FILE MISMATCH BTW EXTENSIONS AND NUM OF FILES")
			sys.exit
		for ref in refNumbers:
			alts[ref] = extensions[refNumbers.index(ref)]

		for ref, ext in alts.items():
			# print("ALT FILE EXTENSION")
			# print(ext)
			# print("ALT FILE REFERENCE NUMBER")
			# print(ref)
			new_parameters = (
				"param1={}"
				"&param2=1"
				"&param3="
				"&param4="
				"&param5={}"
				"&param6="
				"&param7="
				"&param8={}".format(
					self.rsAssetID,
					ext,
					ref
					)
				)
			alternativeAssetPath = self.rsAPI.query(
				"get_resource_path",
				new_parameters,
				self._user
				)
			self.localAssetPaths.append(alternativeAssetPath)

		print("ALL ASSET PATHS:")
		print(self.localAssetPaths)

	def post_to_ia(self):
		'''
		Use the archive.org Python Library to upload an asset to archive.org,
			accompanied by its metadata. Requires the local filepath of the
			asset, as well as a CSV holding the metadata for that asset. The CSV
			must be manually exported and normalized by the operator from a
			ResourceSpace search.
		For more info: archive.org Python Library: https://archive.org/services/docs/api/internetarchive/quickstart.html#metadata
			and archive.org metadata schema: https://archive.org/services/docs/api/metadata-schema/index.html
		'''
		self.get_core_metadata(self.assetMetadata)

		md = {
			# LET'S THINK ABOUT HOW TO MAKE THIS SET OF MD MORE AGNOSTIC/GENERALIZABLE
			'identifier': self.identifier,
			'title': self.title,
			'collection': self.collection,
			# Original CSV columns 'Notes,' 'Digitization QC note,' etc.
			# should be concatenated manually by operator into single column 'Notes'
			'notes': self.assetMetadata['Notes']+"\nDigitized through a generous 2018 Recordings at Risk grant from the Council on Library and Information Resources.",
			# 'notes': self.assetMetadata['Notes']+"\nDigitized through a generous 2018 Humanities Collections and Reference Resources grant from the National Endowment for the Humanities.",
			# Description -> description
			'description': self.assetMetadata['Description'],
			'subject': self.subject,
			'date': self.date,
			# contributor: The person or organization that provided the physical or digital media.
			'contributor': self.assetMetadata['Resource type'],
			'creator': self.creator,
			'language': self.assetMetadata['Language'],
			'rights': rightsStatement,
			#'licenseurl': self.license,
			'coverage': self.assetMetadata['Location of recording'],
			'identifier': self.identifier,
			'condition': self.assetMetadata['Original Material Condition'],
			# Original columns 'Medium of original,' 'Dimensions of original,' 'Original video standard,' 'Generation' columns should be concatenated manually by operator into single column 'Medium of original'
			'source': self.assetMetadata['Medium of original']
		}

		if self.mediaType == 'mp4':
			md['mediatype'] = 'movies'
			# 'frame rate' column should be normalized into numbers manually by operator
			md['frames_per_second'] = self.assetMetadata['Frame rate']
			# 'video size' column should be split into 'Video height' and 'Video width' numbers manually by operator
			md['source_pixel_width'] = self.assetMetadata['Video height']
			md['source_pixel_height'] = self.assetMetadata['Video width'],
			# 'PFA full accession number' column should be normalized to 'urn:bampfa_accession_number:XXXX' manually by operator
			md['external-identifier'] = self.assetMetadata['PFA full accession number'],
			md['sound'] = self.assetMetadata['PFA item sound characteristics'],
			md['color'] = self.assetMetadata['Color characteristics']
		elif self.mediaType == 'mp3':
			md['mediatype'] = 'audio'

		# get rid of empty values in the md dictionary
		md = {k: v for k, v in md.items() if v not in (None,'')}
		# remove trailing "; " in any of the concatenated fields
		md = {k: (v.rsplit('; ',1)[0] if isinstance(v,str) and v.endswith('; ') else v) for k, v in md.items()}
		print("IDENTIFIER:")
		print(self.identifier)
		print("LOCAL ASSET PATHS:")
		print(self.localAssetPaths)
		print("METADATA DICT:")
		print(md)
		### COMMENTED OUT FOR TESTING
		# from the ia package documentation:
		# r = upload('<identifier>', files=['foo.txt', 'bar.mov'], metadata=md)
		# archive.org Python Library, 'uploading': https://archive.org/services/docs/api/internetarchive/quickstart.html#uploading
		result = False
		uploaded = "Didn't get to upload"
		try:
			r = upload(self.identifier, files=self.localAssetPaths, metadata=md)
			if r[0].status_code == 200:
				uploaded = "Uploaded"
				result = True
		except Exception as e:
			print(e)
			uploaded = "Upload failed"
		print(uploaded)
		return result

	def get_core_metadata(self,assetMetadata):
		'''
		Try to get: Creator, Title, Date, Subject, Identifier
		These values are all potentially in multiple columns,
		so we need to sort through the possible fields.
		'''
		# CREATOR
		directors = assetMetadata['Directors / Filmmakers']
		speakers = assetMetadata['Speaker/Interviewee']
		_creator = assetMetadata['Creator']

		self.creator = ''.join([x+"; " for x in (directors,speakers,_creator) if not x in (None,"")])
		self.creator = self.creator.replace("|"," ; ")
		self.creator = [x.strip() for x in self.creator.split(";")]
		self.creator = [x for x in self.creator if not x in ('',None)]

		# TITLE
		mainTitle = assetMetadata['Title']
		altTitle = assetMetadata['Alternative Title']
		eventTitle = assetMetadata['Event title']
		eventSeries = assetMetadata['Event series']
		pfaFilmSeries = assetMetadata['PFA film series']

		self.title = ''.join([str(x)+"; " for x in (mainTitle,altTitle,eventTitle,eventSeries,pfaFilmSeries) if not x in (None,"")])

		# DATE
		release = assetMetadata['Release Date']
		recordingDate = assetMetadata['Date of recording']
		eventYear = assetMetadata['Event year']
		_date = assetMetadata['Date']

		if release not in (None,""):
			self.date = release
		elif recordingDate not in (None,""):
			self.date = recordingDate
		elif eventYear not in (None,""):
			self.date = eventYear
		elif _date not in (None,""):
			self.date = _date

		# SUBJECT
		titleSubjects = assetMetadata["Subject(s): Film title(s)"].replace("|","; ")
		topics = assetMetadata["Subject(s): Topics(s)"]
		nameTopics = assetMetadata["Subject(s): Names"]

		self.subject = ''.join([str(x)+"; " for x in (titleSubjects,topics,nameTopics) if not x in (None,"")])

		# IDENTIFIER
		try:
			if not assetMetadata['Source canonical" name"'] in [None,'']:
				self.identifier = assetMetadata['Source canonical" name"']
			else:
				self.identifier = os.path.splitext(self.assetMetadata['Access copy filename'])[0]
		except:
			self.identifier = self.assetMetadata['Access copy filename']

def parse_resourcespace_csv(csvPath,_user, mediaType):
	'''
	1. Interpret metadata CSV as a 'key:value' dictionary, using the first row
		as the 'key', using DictReader
	2. Assign the correct CSV row of metadata 'values' to the current asset
	3. Post asset with metadata to archive.org (using post_to_ia, defined above)
	'''
	failed_to_redo = []
	tempCSVpath = "{}_tempCSV{}".format(
		os.path.splitext(csvPath)[0],
		os.path.splitext(csvPath)[1])
	with open(csvPath) as _file:
		records = csv.DictReader(_file)
		for row in records:
			# the Asset class __init__ function defines the asset's rsAssetID, which will be stored in the same CSV row as the rest of the metadata
			currentAsset = Asset(
				assetMetadata=row,
				_user=_user,
				mediaType=mediaType
				)
			# get_local_asset_path uses the rsAssetID to find the local filepath of the asset
			currentAsset.get_local_asset_path()
			try:
				currentAsset.get_local_alternative_asset_paths()
			except:
				pass
			print(
				currentAsset.rsAssetID,
				currentAsset.localAssetPaths
				)
			result = currentAsset.post_to_ia()
			if result:
				del currentAsset
			else:
				failed_to_redo.append(row)

	if len(failed_to_redo) > 0:
		with open(tempCSVpath,'w') as f:
			writer = csv.DictWriter(f,failed_to_redo[0].keys())
			writer.writeheader()
			for record in failed_to_redo:
				print("FAILED TO UPLOAD TO ARCHIVE.ORG:")
				print(record['Resource ID(s)'])
				writer.writerow(record)
		os.remove(csvPath)
		csvPath = tempCSVpath.replace("_tempCSV",'')
		result = csvPath
	else:
		result = False

	return result

def parse_drive_url(url):
	try:
		file_google_id = re.match(
			r"(https:\/\/drive\.google\.com\/file\/d\/)([A-Za-z0-9-_]+)(\/.*)",
			url
			).group(2)
		timestamp = time.strftime("%Y-%m-%dT%H-%M-%S")
		dest_path = 'csvs/'+timestamp+'.csv'
		GoogleDriveDownloader.download_file_from_google_drive(
			file_id=file_google_id,
			dest_path= 'csvs/'+timestamp+'.csv'
			)
		time.sleep(2)
		local_csv_path = dest_path
	except:
		local_csv_path =  None

	print("LOCAL CSV PATH")
	print(local_csv_path)

	return local_csv_path

def define_resourcespace_csv():
	csvPath = input("drag and drop path to CSV with ResourceSpace metadata, or paste a Google Drive link here:")
	if not csvPath.startswith("https://drive"):
		pass
	else:
		csvPath = parse_drive_url(csvPath)
		if not csvPath:
			print("YOU MESSED UP! SOMETHINGS UP WITH THE DRIVE LINK OR THE LOCAL FILEPATH")
			sys.exit()
		else:
			pass

	return csvPath

def main():
	_user = User()
	print("Hello, "+_user.rsUserName)
	csvPath = define_resourcespace_csv()
	mediaType = input("You want audio or video? "
		"Type 'a' for audio or 'v' for video: ")
	if mediaType == 'a':
		mediaType = 'mp3'
		print("YOU CHOSE AUDIO! SUPER! THANKS!")
	elif mediaType == 'v':
		mediaType = 'mp4'
		print("YOU CHOSE VIDEO! SUPER! THANKS!")
	else:
		print("YOU ENTERED AN INVALID MEDIA TYPE! JUST TYPE a OR v")
		sys.exit()
	print(mediaType)
	result = parse_resourcespace_csv(csvPath,_user,mediaType)
	if result != False:
		# i.e., if a csv of records to redo gets returned
		redo = input("Some records failed to load. If you want to redo, "
			"type 'r' and enter, otherwise just hit enter and I will quit.")
		if redo == 'r':
			parse_resourcespace_csv(result,_user,mediaType)
		else:
			print("BYE!")

if __name__ == "__main__":
	main()


from fairseq.models.roberta import RobertaModel

roberta = RobertaModel.from_pretrained(
    'checkpoints_QQP/',
    checkpoint_file='checkpoint_best.pt',
    data_name_or_path='QQP-bin'
)

import torch
label_fn = lambda label: roberta.task.label_dictionary.string(
    torch.LongTensor([label + roberta.task.label_dictionary.nspecial])
)
ncorrect, nsamples = 0, 0
roberta.cuda()
roberta.eval()
evaluatedSoFar = set()
lineNumbers = 0
with open('/u/scr/mhahn/PRETRAINED/GLUE/glue_data/QQP/dev_alternatives_c.tsv') as fin:
  with open('/u/scr/mhahn/PRETRAINED/GLUE/glue_data/QQP/dev_datapoints_predictions_fairseq.tsv', "w") as outFile:
    while True:
        lineNumbers += 1
        try:
           line = next(fin).strip()
        except UnicodeDecodeError:
           print("UnicodeDecodeError", lineNumbers)
           continue
        except StopIteration:
           break
        if line == "#####":
           originalSentences = next(fin).strip() # the original
           separation = [int(x) for x in next(fin).strip().split(" ")][0]+1 # position of separation
           sentences = next(fin).strip().split(" ")
        else:
            continue
        print(sentences, separation)
        sentences = [sentences[:separation], sentences[separation:]]
        assert len(sentences[1]) > 3, (line, separation, sentences)
        for i in range(2):
          sentences[i] = ("".join(sentences[i])).replace("▁", " ").replace("</s>", "").strip()
        assert len(sentences[1]) > 0, (line, separation, sentences)
#        print(sentences)
        if tuple(sentences) in evaluatedSoFar:
           continue
        evaluatedSoFar.add(tuple(sentences))
        if len(evaluatedSoFar) % 100 == 0:
           print(len(evaluatedSoFar), sentences)
        tokens = roberta.encode(sentences[0], sentences[1])
        prediction = roberta.predict('sentence_classification_head', tokens)
        prediction_label = label_fn(prediction.argmax().item())
        prediction = [float(x) for x in prediction.view(-1)]
        print("\t".join([sentences[0], sentences[1], str(prediction[1]), prediction_label]), file=outFile)


"""
Field schemas
"""
from marshmallow import fields, validates_schema, ValidationError

from app import MA
from app.helper.choice_options_validator import (
    check_for_repeated_options,
    validate_repeats_of_choice_options
)
from app.helper.range_validator import validate_range_checkbox, validate_range_text
from app.helper.constants import MAX_FIELD_TYPE, MIN_FIELD_TYPE
from app.helper.enums import FieldType
from app.schemas.range import RangeSchema
from app.schemas.setting_autocomplete import SettingAutocompleteSchema

class BasicField(MA.Schema):
    """
    Basic field schema (also used as TextAreas Schema)
    """

    class Meta:
        """
        Basic field schema meta
        """
        fields = ("id", "owner_id", "name", "field_type", "created")

    name = fields.Str(required=True)
    owner_id = fields.Integer(data_key="ownerId")
    field_type = fields.Integer(required=True, data_key="fieldType")

class FieldPostSchema(BasicField):
    """
    Field schema
    """

    class Meta:
        """
        Field schema meta
        """
        fields = ("owner_id", "name", "field_type", "is_strict", "range",
                  "setting_autocomplete", "choice_options")

    is_strict = fields.Boolean(data_key='isStrict')
    range = fields.Nested(RangeSchema)
    setting_autocomplete = fields.Nested(SettingAutocompleteSchema, data_key="settingAutocomplete")
    choice_options = fields.List(fields.Str(required=False), data_key="choiceOptions")

    @validates_schema
    # pylint: disable=no-self-use
    def validate_field_type(self, data, **kwargs):
        """
        Validates incoming field type, and raises error if type is greater then 6 or lower than 1
        :param data:
        :param kwargs:
        :return: None or raise error
        """
        if data.get('field_type') > MAX_FIELD_TYPE or data.get('field_type') < MIN_FIELD_TYPE:
            raise ValidationError(
                {'fieldType': {
                    '_schema': 'Must be greater than or equal to 1 and less than or equal to 6.'
                }}
            )

    @validates_schema
    # pylint: disable=no-self-use
    def validate_choice_options(self, data, **kwargs):
        """
        Validates if choice options has repeated values
        :param data:
        :param kwargs:
        :return:
        """
        if data.get('field_type') in (FieldType.Radio.value, FieldType.Checkbox.value):
            errors = []
            options = data.get('choice_options')
            if not options and isinstance(options, list):
                errors.append('choiceOptions is empty')
            if check_for_repeated_options(options=options):
                errors.append('Repeated values')
            if errors:
                raise ValidationError({'choiceOptions': {'_schema': errors}})

    @validates_schema
    # pylint: disable=no-self-use
    def validate_range_values(self, data, **kwargs):
        """
        Validates range for text and checkbox fields
        :param data:
        :param kwargs:
        :return:
        """
        field_type = data.get('field_type')
        ranges = data.get('range')
        errors = []
        if ranges:
            range_min = ranges.get('min')
            range_max = ranges.get('max')
            if field_type == FieldType.Text.value:
                errors.extend(validate_range_text(range_min, range_max))
            if field_type == FieldType.Checkbox.value:
                errors.extend(validate_range_checkbox(range_min, range_max))
        if errors:
            raise ValidationError({'range': {'_schema': errors}})

class FieldCheckboxSchema(BasicField):
    """
    Field with checkbox options and optional range
    """

    class Meta:
        """
        Field Checkbox Schema
        """
        fields = ("owner_id", "name", "field_type", "choice_options", "range")

    choice_options = fields.List(fields.Str(), required=True, data_key="choiceOptions")
    range = fields.Nested(RangeSchema, required=False)

    @validates_schema
    # pylint:disable=no-self-use
    def validate_range_of_choices(self, data, **kwargs):
        """
        Validates range to choice_options amount
        :param data:
        :param kwargs:
        :return: if raised returns
        {"message":{"range":{"_schema": [errors] }}}
        """
        range_dict = data.get('range')
        options_list = data.get('choice_options')
        errors = []
        if range_dict:
            min_value = range_dict.get('min')
            max_value = range_dict.get('max')
            if min_value is not None:
                if min_value < 0:
                    errors.append('Min selective options must be positive')
                if min_value > len(options_list):
                    errors.append('Min selective options must be less than list of options')
            if max_value is not None:
                if max_value < 0:
                    errors.append('Max selective options must be positive')
                if max_value > len(options_list):
                    errors.append('Max selective options must be less than list of options')
        if errors:
            raise ValidationError({'range': {'_schema': errors}})

class FieldRadioSchema(BasicField):
    """
    Field with choice options schema
    """

    class Meta:
        """
        Field with choice options schema meta
        """
        fields = ("owner_id", "name", "field_type", "choice_options")

    choice_options = fields.List(fields.Str(), required=True, data_key="choiceOptions")

class FieldNumberTextSchema(BasicField):
    """
    Field with type number or text schema
    """

    class Meta:
        """
        Field with type number or text schema meta
        """
        fields = ("id", "owner_id", "name", "field_type", "range", "is_strict")

    is_strict = fields.Boolean(required=False, data_key="isStrict")
    range = fields.Nested(RangeSchema, required=False)

class FieldSettingAutocompleteSchema(BasicField):
    """
    Field with autocomplete settings schema
    """

    class Meta:
        """
        Field with autocomplete settings schema meat
        """
        fields = ("owner_id", "name", "field_type", "setting_autocomplete")

    setting_autocomplete = fields.Nested(
        SettingAutocompleteSchema,
        required=True,
        data_key="settingAutocomplete"
    )

class FieldPutSchema(BasicField):
    """
    Field put schema
    """

    class Meta:
        """
        Field put schema meta
        """
        fields = (
            "updated_name",
            "is_strict",
            "range",
            "added_choice_options",
            "removed_choice_options",
            "updated_autocomplete",
            "delete_range"
        )

    updated_name = fields.Str(required=False, data_key="updatedName")
    range = fields.Nested(RangeSchema)
    added_choice_options = fields.List(
        fields.Str(),
        required=False,
        data_key="addedChoiceOptions"
    )
    removed_choice_options = fields.List(
        fields.Str(),
        required=False,
        data_key="removedChoiceOptions"
    )
    updated_autocomplete = fields.Nested(
        SettingAutocompleteSchema,
        required=False,
        data_key="updatedAutocomplete"
    )
    delete_range = fields.Bool(required=False, data_key="deleteRange")
    is_strict = fields.Bool(required=False, data_key="isStrict")

    @validates_schema
    # pylint:disable=no-self-use
    def validate_new_or_deleted_range(self, data, **kwargs):
        """
        Validates if trying to update and delete range in same field
        :param data:
        :param kwargs:
        :return:
        """
        new_range = data.get('range')
        delete_range = data.get('delete_range')
        if new_range and delete_range:
            raise ValidationError({'range': {'_schema': 'Can\'t update and delete range'}})

class FieldNumberTextPutSchema(BasicField):
    """
    Field with type number or text schema on put request
    """

    class Meta:
        """
        Fields for field of type number or text schema meta on put request
        """
        fields = ("updated_name", "range", "is_strict", "delete_range")

    updated_name = fields.Str(required=False, data_key="updatedName")
    is_strict = fields.Boolean(required=False, data_key="isStrict")
    range = fields.Nested(RangeSchema, required=False)
    delete_range = fields.Boolean(required=False, data_key="deleteRange")

class FieldRadioPutSchema(BasicField):
    """
    Schema for field with radio type to use on put request
    """

    class Meta:
        """
        Fields for schema of field with radio type to use on put request
        """
        fields = ("updated_name", "added_choice_options", "removed_choice_options")

    updated_name = fields.Str(required=False, data_key="updatedName")
    added_choice_options = fields.List(
        fields.Str(),
        required=False,
        data_key="addedChoiceOptions"
    )
    removed_choice_options = fields.List(
        fields.Str(),
        required=False,
        data_key="removedChoiceOptions"
    )

    @validates_schema
    # pylint:disable=no-self-use
    def validate_choice_options(self, data, **kwargs):
        """
        Validates if in added or removed are repeatable values
        :param data:
        :param kwargs:
        :return:
        """
        errors = validate_repeats_of_choice_options(
            added=data.get('added_choice_options'),
            removed=data.get('removed_choice_options'))
        if errors:
            raise ValidationError({'choiceOptions': {'_schema': errors}})

class FieldCheckboxPutSchema(BasicField):
    """
    Schema for a field with checkbox type to use on put request
    """

    class Meta:
        """
        Fields for schema of field with checkbox type to use on put request
        """
        fields = (
            "updated_name",
            "range",
            "added_choice_options",
            "removed_choice_options",
            "delete_range"
        )

    updated_name = fields.Str(required=False, data_key="updatedName")
    range = fields.Nested(RangeSchema, required=False)
    added_choice_options = fields.List(
        fields.Str(),
        required=False,
        data_key="addedChoiceOptions"
    )
    removed_choice_options = fields.List(
        fields.Str(),
        required=False,
        data_key="removedChoiceOptions"
    )
    delete_range = fields.Boolean(data_key='deleteRange')

    @validates_schema
    # pylint:disable=no-self-use
    def validate_choice_options(self, data, **kwargs):
        """
        Validates if in added or removed are repeatable values
        :param data:
        :param kwargs:
        :return:
        """
        errors = validate_repeats_of_choice_options(
            added=data.get('added_choice_options'),
            removed=data.get('removed_choice_options'))
        if errors:
            raise ValidationError({'choiceOptions': {'_schema': errors}})

    @validates_schema
    # pylint:disable=no-self-use
    def validate_checkbox_range(self, data, **kwargs):
        """
        Validates checkbox range
        :param data:
        :param kwargs:
        :return:
        """
        ranges = data.get('range')
        errors = []
        if ranges:
            range_min = ranges.get('min')
            range_max = ranges.get('max')
            errors.extend(validate_range_checkbox(range_min, range_max))
        if errors:
            raise ValidationError({'range': {'_schema': errors}})

class FieldAutocompletePutSchema(BasicField):
    """
    Schema for a field with autocomplete type to use on put request
    """

    class Meta:
        """
        Fields for schema of field with autocomplete type to use on put request
        """
        fields = ("updated_name", "updated_autocomplete")

    updated_name = fields.Str(required=False, data_key="updatedName")
    updated_autocomplete = fields.Nested(
        SettingAutocompleteSchema,
        required=False,
        data_key="updatedAutocomplete"
    )

class FieldTextAreaPutSchema(BasicField):
    """
    Schema for a field with textarea type to use on put request
    """

    class Meta:
        """
        Fields for schema of field with textarea type to use on put request
        """
        fields = ("updated_name",)

    updated_name = fields.Str(required=False, data_key="updatedName")


from __future__ import absolute_import
from __future__ import unicode_literals

import unittest

import mock
from pytest import raises

from paasta_tools.deployd.common import BounceTimers
from paasta_tools.deployd.common import ServiceInstance
from paasta_tools.deployd.workers import PaastaDeployWorker
from paasta_tools.marathon_tools import DEFAULT_SOA_DIR

class TestPaastaDeployWorker(unittest.TestCase):
    def setUp(self):
        self.mock_inbox_q = mock.Mock()
        self.mock_bounce_q = mock.Mock()
        self.mock_metrics = mock.Mock()
        mock_config = mock.Mock(
            get_cluster=mock.Mock(return_value='westeros-prod'),
            get_deployd_worker_failure_backoff_factor=mock.Mock(return_value=30)
        )
        with mock.patch(
            'paasta_tools.deployd.workers.PaastaDeployWorker.setup', autospec=True
        ):
            self.worker = PaastaDeployWorker(1,
                                             self.mock_inbox_q,
                                             self.mock_bounce_q,
                                             mock_config,
                                             self.mock_metrics)

    def test_setup(self):
        with mock.patch(
            'paasta_tools.deployd.workers.marathon_tools.load_marathon_config', autospec=True
        ), mock.patch(
            'paasta_tools.deployd.workers.marathon_tools.get_marathon_client', autospec=True
        ):
            self.worker.setup()

    def test_setup_timers(self):
        mock_si = mock.Mock(bounce_timers=None,
                            service='universe',
                            instance='c137')
        ret = self.worker.setup_timers(mock_si)
        calls = [mock.call('bounce_length_timer',
                           service='universe',
                           paasta_cluster='westeros-prod',
                           instance='c137'),
                 mock.call().start(),
                 mock.call('processed_by_worker',
                           service='universe',
                           paasta_cluster='westeros-prod',
                           instance='c137'),
                 mock.call('setup_marathon_timer',
                           service='universe',
                           paasta_cluster='westeros-prod',
                           instance='c137')]
        self.mock_metrics.create_timer.assert_has_calls(calls)
        assert ret == BounceTimers(processed_by_worker=self.mock_metrics.create_timer.return_value,
                                   setup_marathon=self.mock_metrics.create_timer.return_value,
                                   bounce_length=self.mock_metrics.create_timer.return_value)

    def test_run(self):
        with mock.patch(
            'time.sleep', autospec=True
        ) as mock_sleep, mock.patch(
            'paasta_tools.deployd.workers.marathon_tools.get_all_marathon_apps', autospec=True
        ) as mock_get_all_marathon_apps, mock.patch(
            'paasta_tools.deployd.workers.PaastaDeployWorker.setup_timers', autospec=True
        ) as mock_setup_timers, mock.patch(
            'paasta_tools.deployd.workers.deploy_marathon_service', autospec=True
        ) as mock_deploy_marathon_service, mock.patch(
            'time.time', autospec=True, return_value=1
        ):
            mock_sleep.side_effect = LoopBreak
            self.worker.marathon_client = mock.Mock()
            self.worker.marathon_config = mock.Mock()
            mock_deploy_marathon_service.return_value = (0, None)
            mock_si = mock.Mock(service='universe',
                                instance='c137',
                                failures=0)
            self.mock_bounce_q.get.return_value = mock_si
            with raises(LoopBreak):
                self.worker.run()
            mock_setup_timers.assert_called_with(self.worker, mock_si)
            assert mock_setup_timers.return_value.setup_marathon.start.called
            mock_deploy_marathon_service.assert_called_with(service='universe',
                                                            instance='c137',
                                                            client=self.worker.marathon_client,
                                                            soa_dir=DEFAULT_SOA_DIR,
                                                            marathon_config=self.worker.marathon_config,
                                                            marathon_apps=mock_get_all_marathon_apps.return_value)
            assert mock_setup_timers.return_value.setup_marathon.stop.called
            assert not mock_setup_timers.return_value.processed_by_worker.start.called
            assert not self.mock_inbox_q.put.called
            assert mock_setup_timers.return_value.bounce_length.stop.called

            mock_deploy_marathon_service.return_value = (0, 60)
            mock_setup_timers.return_value.bounce_length.stop.reset_mock()
            with raises(LoopBreak):
                self.worker.run()
            mock_setup_timers.assert_called_with(self.worker, mock_si)
            assert mock_setup_timers.return_value.setup_marathon.start.called
            mock_deploy_marathon_service.assert_called_with(service='universe',
                                                            instance='c137',
                                                            client=self.worker.marathon_client,
                                                            soa_dir=DEFAULT_SOA_DIR,
                                                            marathon_config=self.worker.marathon_config,
                                                            marathon_apps=mock_get_all_marathon_apps.return_value)
            assert mock_setup_timers.return_value.setup_marathon.stop.called
            assert mock_setup_timers.return_value.processed_by_worker.start.called
            self.mock_inbox_q.put.assert_called_with(ServiceInstance(service='universe',
                                                                     instance='c137',
                                                                     bounce_by=61,
                                                                     watcher='Worker1',
                                                                     bounce_timers=mock_setup_timers.return_value,
                                                                     failures=0))
            assert not mock_setup_timers.return_value.bounce_length.stop.called

            mock_deploy_marathon_service.side_effect = Exception()
            mock_setup_timers.return_value.bounce_length.stop.reset_mock()
            with raises(LoopBreak):
                self.worker.run()
            mock_setup_timers.assert_called_with(self.worker, mock_si)
            assert mock_setup_timers.return_value.setup_marathon.start.called
            mock_deploy_marathon_service.assert_called_with(service='universe',
                                                            instance='c137',
                                                            client=self.worker.marathon_client,
                                                            soa_dir=DEFAULT_SOA_DIR,
                                                            marathon_config=self.worker.marathon_config,
                                                            marathon_apps=mock_get_all_marathon_apps.return_value)
            assert mock_setup_timers.return_value.setup_marathon.stop.called
            assert mock_setup_timers.return_value.processed_by_worker.start.called
            self.mock_inbox_q.put.assert_called_with(ServiceInstance(service='universe',
                                                                     instance='c137',
                                                                     bounce_by=61,
                                                                     watcher='Worker1',
                                                                     bounce_timers=mock_setup_timers.return_value,
                                                                     failures=1))
            assert not mock_setup_timers.return_value.bounce_length.stop.called

class LoopBreak(Exception):
    pass


import torch
from torch.autograd import Variable
from torch.nn import Module
import h5py

import numpy as np

from allennlp.data.dataset import Dataset
from allennlp.data import Token, Vocabulary, Instance
from allennlp.data.fields import TextField
from allennlp.data.token_indexers.elmo_indexer import ELMoTokenCharactersIndexer
from allennlp.nn.util import remove_sentence_boundaries
from allennlp.modules.elmo import _ElmoBiLm

from chunking.data import variableFromSentence

options_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_options.json"
weight_file = "https://s3-us-west-2.amazonaws.com/allennlp/models/elmo/2x4096_512_2048cnn_2xhighway/elmo_2x4096_512_2048cnn_2xhighway_weights.hdf5"
elmo_bilm = _ElmoBiLm(options_file, weight_file).cuda()

indexer = ELMoTokenCharactersIndexer()

__all__ = ['elmo_bilm', 'embed_sentence', 'ElmoEmbedder', 'variablesFromPairElmo', 'elmo_variable_from_sentence']

use_cuda = torch.cuda.is_available()

class ElmoEmbedder(Module):
    def __init__(self, elmo_bilm, special_tokens, device):
        super(ElmoEmbedder, self).__init__()
        self.elmo_bilm = elmo_bilm
        self.device = device
        self.special_tokens = special_tokens
        self.elmo_id_to_special_token = {tuple(token_to_elmo_id(tok).view(-1).data): tok_id for (tok_id, tok) in enumerate(special_tokens)}
        self.special_token_to_id = {tok: i for (i, tok) in enumerate(special_tokens)}
        self.dimension = len(special_tokens) + 1024
        self.cached_embedding = (-1, None)

    def forward(self, input, token_index):
        input_code = tuple([tuple(input[0][i].data) for i in range(input.shape[1])])
        token_code = tuple(input[0][token_index].data)
        if token_code in self.elmo_id_to_special_token:
            tok_id = self.elmo_id_to_special_token[token_code]
            result = Variable(torch.from_numpy(np.eye(self.dimension)[tok_id]).float())
        elif input_code == self.cached_embedding[0]:
            embedded = self.cached_embedding[1]
            result = torch.cat([Variable(torch.from_numpy(np.zeros(len(self.special_tokens))).float()),
                                Variable(embedded[token_index])])
        else:
            embedded = embed_numerical_sent(input, elmo_bilm, self.device)
            self.cached_embedding = (input_code, embedded)
            result = torch.cat([Variable(torch.from_numpy(np.zeros(len(self.special_tokens))).float()),
                                Variable(embedded[token_index])])
        if use_cuda:
            return result.cuda()
        else:
            return result

def character_ids_to_embeddings(character_ids, elmo_bilm, device):
    # returns (batch_size, 3, num_times, 1024) embeddings and (batch_size, num_times) mask
    if device >= 0:
        character_ids = character_ids.cuda(device=device)
    bilm_output = elmo_bilm(character_ids)
    layer_activations = bilm_output['activations']
    mask_with_bos_eos = bilm_output['mask']
    without_bos_eos = [remove_sentence_boundaries(layer, mask_with_bos_eos)
            for layer in layer_activations]
    # without_bos_eos is a 3 element list of (batch_size, num_times, dim) arrays
    activations = torch.cat([ele[0].unsqueeze(1) for ele in without_bos_eos], dim=1)
    mask = without_bos_eos[0][1]
    return activations, mask

def embed_numerical_sent(input_var, elmo_bilm, device):
    embeddings, mask = character_ids_to_embeddings(input_var, elmo_bilm, device)
    sent_embeddings = []
    for i in range(1):
        length = int(mask[i, :].sum())
        sentence_embedding = embeddings[i, :, :length, :].data.cpu().numpy()
        a = torch.from_numpy(sentence_embedding[0])
        b = torch.from_numpy(sentence_embedding[1])
        c = torch.from_numpy(sentence_embedding[2])
        result = (a + b + c) / 3.0
        sent_embeddings.append(result)
    return sent_embeddings[0]

def token_to_elmo_id(token):
    tokens = [Token(token)]
    field = TextField(tokens, {'character_ids': indexer})
    instance = Instance({"elmo": field})
    instances = [instance]
    dataset = Dataset(instances)
    vocab = Vocabulary()
    for instance in dataset.instances:
        instance.index_fields(vocab)
    #dataset.index_instances(vocab) # replaced by above, so that there's no progress bar
    return dataset.as_tensor_dict()['elmo']['character_ids']

def batch_to_ids(batch):
    """
    Given a batch (as list of tokenized sentences), return a batch
    of padded character ids.
    """
    instances = []
    for sentence in batch:
        tokens = [Token(token) for token in sentence]
        field = TextField(tokens, {'character_ids': indexer})
        instance = Instance({"elmo": field})
        instances.append(instance)

    dataset = Dataset(instances)
    vocab = Vocabulary()
    for instance in dataset.instances:
        instance.index_fields(vocab)
    #dataset.index_instances(vocab)        #replaced by above, so there's no progress bar
    return dataset.as_tensor_dict()['elmo']['character_ids']

def elmo_variable_from_sentence(sent):
    tokens = sent.split()
    tokens.append('eos')
    return batch_to_ids([tokens])

# Maps the source and target sentence into PyTorch vectors (of the token ids).
def variablesFromPairElmo(pair, output_lang):
    input_variable = elmo_variable_from_sentence(pair[0])
    target_variable = variableFromSentence(output_lang, pair[1])
    return (input_variable, target_variable)

def pad(tokens, desired_length, padder):
    padding = [padder] * (desired_length - len(tokens))
    return tokens + padding

def elmo_variable_from_sentences(sents):
    tokens = [sent.split() for sent in sents]
    max_size = max([len(x) for x in tokens]) + 1
    padded = [pad(tok, max_size, padder='eos') for tok in tokens]
    return batch_to_ids(padded)


#!/usr/bin/python3
"""
Create view route for states
"""
from api.v1.views import app_views
from flask import Flask, jsonify, abort, request
from models import storage
from models.state import State

@app_views.route('/states', strict_slashes=False, methods=['GET'])
def list_states():
    """list states"""
    list_states = []
    get_state = storage.all(State)
    for k, v in get_state.items():
        list_states.append(v.to_dict())
    return jsonify(list_states)

@app_views.route('/states/<state_id>', strict_slashes=False, methods=['GET'])
def state(state_id=None):
    """list states"""
    get_state = {}
    if state_id:
        get_state = storage.get(State, state_id)

    if get_state is None:
        return abort(404)

    return jsonify(get_state.to_dict()), 200

@app_views.route('/states/<state_id>', strict_slashes=False,
                 methods=['DELETE'])
def del_state(state_id=None):
    """delete states"""

    get_state = storage.get(State, state_id)
    if get_state:
        storage.delete(get_state)
        storage.save()
        return jsonify({}), 200
    else:
        return abort(404)

@app_views.route('/states', strict_slashes=False, methods=['POST'])
def add_state():
    """add state"""
    if not request.get_json(silent=True):
        return jsonify({'error': 'Not a JSON'}), 400

    if 'name' not in request.json:
        abort(400, 'Missing name')

    new_s = State()
    data = dict(request.get_json(silent=True))
    new_s.name = data['name']

    storage.new(new_s)
    storage.save()
    storage.reload()

    return jsonify(new_s.to_dict()), 201

@app_views.route('/states/<state_id>', strict_slashes=False, methods=['PUT'])
def update_state(state_id=None):
    """update states"""

    data = request.get_json(silent=True)
    if not data:
        return jsonify({'error': 'Not a JSON'}), 400

    get_state = storage.get(State, state_id)

    if get_state:
        update_s = dict(request.get_json(silent=True))
        for k, v in update_s.items():
            setattr(get_state, k, v)

        storage.save()
    else:
        abort(404)

    return jsonify(get_state.to_dict()), 200


from core.gui import PATCH_COLS, PATCH_ROWS
from core.world_patch_block import World, Patch
from core.agent import Agent
from pygame.color import Color
from random import randint
from core.sim_engine import SimEngine

class Braess_Road_Patch(Patch):
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.road_type = None
        self.delay = None

    def set_road_type(self, type):
        self.road_type = type

    def set_delay(self, type):
        self.delay = type

class Commuter(Agent):
    def __init__(self, route, **kwargs):
        super().__init__(route, **kwargs)
        self.birth_tick = None
        self.ticks = 1
        self.route = route

    def count_tick(self):
        self.ticks = self.ticks + 1

    def set_route(self, choice):
        self.route = choice

    def move(self, turn=True, patch=True, end=True):
        if patch == turn:
            self.face()
        if patch == end:
            self.end_trip()

    def face(self, route):
        super.face()

    def end_trip(self):
        super.delete(self)

    def delete(self):
        World.agents.remove(self)
        World.links -= {lnk for lnk in World.links if lnk.includes(self)}

class Braess_Road_World(World):

    def __init__(self, *args, **kwargs):

        super().__init__(*args, **kwargs)

        self.latest_travel_time = None
        self.latest_top_time = None
        self.latest_middle_time = None

        self.spawn_time = None
        self.avg_time = None

        self.cars_spawned = None

        self.top_prob = None
        self.bottom_prob = None
        self.middle_prob = None

        self.top_left_patch = None
        self.top_right_patch = None
        self.bottom_left_patch = None
        self.bottom_right_patch = None

        self.congestion_top_road = None
        self.congestion_bottom_road = None

        # world class might have a ticks attribute
        self.ticks = None
        self.middle_on = False

    def setup(self):
        # Clear everything
        self.reset_all()

        # Set the corner patches

        self.top_left_patch: Patch = World.patches_array[2][2]
        self.top_right_patch: Patch = World.patches_array[2][PATCH_COLS - 3]
        self.bottom_left_patch: Patch = World.patches_array[PATCH_ROWS - 3][PATCH_COLS - 3]
        self.bottom_right_patch: Patch = World.patches_array[PATCH_ROWS - 3][2]

        # Set up the roads
        self.setup_roads()

    def setup_roads(self):
        # 1. Cover everything in random green grass (set all patches a random green color)
        for patch in self.patches:
            color = randint(0, 16) + 96
            patch.set_color(Color(0, color, 0))

        self.draw_roads()

    def draw_roads(self):
        # 2. Set colors and properties on all corners
        self.top_left_patch.set_color(Color('Green'))
        self.top_right_patch.set_color(Color('Blue'))
        self.bottom_right_patch.set_color(Color('Blue'))
        self.bottom_left_patch.set_color(Color('Red'))

        corner_padding = self.bottom_left_patch.neighbors_8() + self.bottom_right_patch.neighbors_8() + \
                         self.top_left_patch.neighbors_8() + self.top_right_patch.neighbors_8()
        # print(neighbors)
        for p in corner_padding:
            p.set_color(Color('Grey'))

        # set up top and bottom roads, set the road types
        for i in range(3, PATCH_COLS - 3):

            # top road
            road_type = 1
            self.patches_array[1, i].set_color(Color('Grey'))
            self.patches_array[1, i].set_road_type(road_type)
            self.patches_array[3, i].set_color(Color('Grey'))
            self.patches_array[3, i].set_road_type(road_type)
            self.patches_array[2, i].set_color(Color('Yellow'))
            self.patches_array[2, i].set_road_type(road_type)
            # bottom road
            self.patches_array[PATCH_ROWS - 2, i].set_color(Color('Grey'))
            self.patches_array[PATCH_ROWS - 2, i].set_road_type(road_type)
            self.patches_array[PATCH_ROWS - 4, i].set_color(Color('Grey'))
            self.patches_array[PATCH_ROWS - 4, i].set_road_type(road_type)
            self.patches_array[PATCH_ROWS - 3, i].set_color(Color('Yellow'))
            self.patches_array[PATCH_ROWS - 3, i].set_road_type(road_type)

            # left road
            self.patches_array[i, 1].set_color(Color('Grey'))
            self.patches_array[i, 1].set_road_type(road_type)
            self.patches_array[i, 3].set_color(Color('Grey'))
            self.patches_array[i, 3].set_road_type(road_type)
            self.patches_array[i, 2].set_color(Color('Yellow'))
            self.patches_array[i, 2].set_road_type(road_type)

            # Right Road
            self.patches_array[i, PATCH_ROWS - 2].set_color(Color('Grey'))
            self.patches_array[i, PATCH_ROWS - 2].set_road_type(road_type)
            self.patches_array[i, PATCH_ROWS - 4].set_color(Color('Grey'))
            self.patches_array[i, PATCH_ROWS - 4].set_road_type(road_type)
            self.patches_array[i, PATCH_ROWS - 3].set_color(Color('Yellow'))
            self.patches_array[i, PATCH_ROWS - 3].set_road_type(road_type)

            if SimEngine.gui_get(MIDDLE_ON):
                # self.patches_array[PATCH_ROWS - ,]
                # [PATCH_ROWS - 3][2]
                i = 0
                for i in range(44):
                    print(i)
                    x = PATCH_ROWS - 4 - i
                    y = 3 + i
                    diagonal_patch = self.patches_array[x][y]
                    top_patch = self.patches_array[x - 1][y]
                    left_patch = self.patches_array[x][y + 1]
                    diagonal_patch.set_color(Color('Orange'))
                    top_patch.set_color(Color('Grey'))
                    left_patch.set_color(Color('Grey'))
                    if i == 43:
                        self.patches_array[x - 1][y + 1].set_color(Color('Orange'))
                # set the last patch

# ############################################## Define GUI ############################################## #
import PySimpleGUI as sg

MIDDLE_ON = 'middle_on'

# switches = [sg.CB(n + '\n 1', key=n, pad=((30, 0), (0, 0)), enable_events=True)
#                                              for n in reversed(CA_World.bin_0_to_7)]
gui_left_upper = [[sg.Text('Middle On?', pad=((0, 5), (20, 0))), sg.CB('True', key=MIDDLE_ON, pad=((0, 5), (20, 0)))]]

gui_left_upperw = [[sg.Text('Middle on', pad=((0, 5), (20, 0))),
                    sg.Slider(key='nbr_agents', range=(1, 101), resolution=25, default_value=25,
                              orientation='horizontal', size=(10, 20))]]

if __name__ == "__main__":
    from core.agent import PyLogo

    # PyLogo(Braess_Road_World, 'Braess Road Paradox', gui_left_upper, bounce=True, patch_size=9, board_rows_cols=(71, 71))
    PyLogo(world_class=Braess_Road_World, caption='Braess Road Paradox', agent_class=Commuter,
           gui_left_upper=gui_left_upper, patch_class=Braess_Road_Patch)


"""Unit tests for tech_support_show.py."""

import pytest

from photon.backend.cisco import tech_support_show
from photon.lib import cisco_utils
from photon.lib import test_utils

#pylint: disable=redefined-outer-name,protected-access

@pytest.fixture(scope='module')
def support_show_parser():
    """Return a switch object fixture."""
    logfile = test_utils.get_files_of_type('cisco_test_log')[0]
    return tech_support_show.SupportShowParser(logfile)

def test_all_parsers(support_show_parser):
    """Test that every getter has a test."""
    test_utils.all_parsers_have_test(support_show_parser, __file__)

def test__get_first_field_val(support_show_parser):
    """Test that we get the first field successfully from show_switchname."""
    result = support_show_parser._get_first_field_val('show_switchname')
    expected = 'MFXRFPEMC9513-A'
    assert result == expected

def test__get_regex_from_output(support_show_parser):
    """Test that we get a regex dict from an output."""
    result = support_show_parser._get_regex_from_output('show_version', 'BIOS:\s+version\s+(?P<bios>.*)')
    assert result == [{'bios': '1.0.10'}]

def test_get_all_fields(support_show_parser):
    """Test that we get all fields successfully."""
    result = set(support_show_parser.get_all_fields().keys())
    assert result == set(list(support_show_parser.fields) + ['Timestamp'])

def test_get_bios(support_show_parser):
    """Test that we get a bios version from a log."""
    result = support_show_parser.get_bios()
    assert result == '1.0.10'

def test_get_fields(support_show_parser):
    """Test that we get some fields properly."""
    result = support_show_parser.get_fields(['show_clock', 'show_switchname'])
    expected = {
        'show_clock': ['16:04:11.752 UTC Thu Jun 14 2018\n', 'Time source is NTP\n'],
        'show_switchname': ['MFXRFPEMC9513-A\n'],
        'Timestamp': '16:04:11.752 UTC Thu Jun 14 2018'
    }
    assert result == expected

def test_get_alias_dict(support_show_parser):
    """Test that we get an alias dict from a log."""
    result = support_show_parser.get_alias_dict()
    assert len(result.keys()) == 136
    assert result['cvrpuxsqp01a_ql'] == '21:01:00:1b:32:3a:f0:a2'
    assert result['21:01:00:1b:32:3a:f0:a2'] == 'cvrpuxsqp01a_ql'

def test_get_flogi_dict(support_show_parser):
    """Test that we get a flogi dict from a log."""
    result = support_show_parser.get_flogi_dict()
    assert result['fc1/18'] == [
        cisco_utils.FlogiEntry(interface='fc1/18', vsan='50', fcid='0x3202a0', port_name='10:00:00:90:fa:1b:32:38', node_name='20:00:00:90:fa:1b:32:38', alias=None)
    ]
    assert result['port-channel1'][0:2] == [
        cisco_utils.FlogiEntry(interface='port-channel1', vsan='50', fcid='0x3200a1', port_name='20:01:00:25:b5:a0:a1:2f', node_name='20:00:00:25:b5:a0:01:2f', alias=None),
        cisco_utils.FlogiEntry(interface='port-channel1', vsan='50', fcid='0x3200a2', port_name='20:01:00:25:b5:a0:a1:3f', node_name='20:00:00:25:b5:a0:01:3f', alias=None)
    ]

def test_get_hardware(support_show_parser):
    """Test that we get hardware model from a log."""
    result = support_show_parser.get_hardware()
    assert result == 'cisco MDS 9513 (13 Slot) Chassis ("Supervisor/Fabric-2a")'

def test_get_interface_dict(support_show_parser):
    """Test that we get interface dicts from the log."""
    result = support_show_parser.get_interface_dict()
    expected = {
        '5_minutes_input_bits_sec': '115810464',
        '5_minutes_output_bits_sec': '195751936',
        'admin_fec_state': 'down',
        'admin_port_mode': 'auto',
        'connected_wwpns': ['20:01:00:25:b5:a0:a1:2f',
        '20:01:00:25:b5:a0:a1:3f',
        '20:01:00:25:b5:a0:a1:0f',
        '20:01:00:25:b5:a0:a1:1f',
        '20:01:00:25:b5:a0:a1:2e',
        '20:01:00:25:b5:a0:a1:3e',
        '20:01:00:25:b5:a0:a1:1e',
        '20:01:00:25:b5:a0:a1:0e',
        '24:01:00:05:73:d2:59:c0'],
        'error': 'port not present',
        'hardware': 'Fibre Channel',
        'input_bytes': '293149753678128',
        'input_discards': '0',
        'input_errors': '0',
        'input_frames_input': '181200169488',
        'input_input_ols': '0',
        'input_invalid_crc/fcs': '0',
        'input_loop_inits': '0',
        'input_lrr': '0',
        'input_nos': '0',
        'input_too_long': '0',
        'input_too_short': '0',
        'input_unknown_class': '0',
        'interface_last_changed': 'Sun Jan 26 18:23:15 2014',
        'interface_name': 'port-channel1',
        'member_1': 'fc1/14',
        'member_2': 'fc1/15',
        'member_3': 'fc1/32',
        'member_4': 'fc1/33',
        'oper_fec_state': 'down',
        'output_bytes': '501873283644288',
        'output_discards': '28755',
        'output_errors': '0',
        'output_frames_output': '302772057121',
        'output_loop_inits': '0',
        'output_lrr': '0',
        'output_nos': '0',
        'output_output_ols': '0',
        'port_mode': 'F',
        'port_vsan': '50',
        'port_wwn': '24:01:00:0d:ec:3c:06:00',
        'snmp_link_state_traps': 'enabled',
        'speed': '32 Gbps',
        'state': 'up',
        'trunk_mode': 'off'
    }
    assert result['port-channel1'] == expected

def test_get_show_clock(support_show_parser):
    """Test that we get show clock output from a log."""
    support_show_parser.get_fields(['show_clock'])
    result = support_show_parser.get_fields(['show_clock'])
    expected = ['16:04:11.752 UTC Thu Jun 14 2018\n', 'Time source is NTP\n']
    assert expected == result['show_clock']
    assert len(result['show_clock']) == 2

def test_get_show_device_alias_database(support_show_parser):
    """Test that we get show device alias database output from a log."""
    result = support_show_parser.get_fields(['show_device_alias_database'])
    expected_start = [
        'device-alias name PURE_CT0_FC0 pwwn 52:4a:93:77:59:62:b2:00\n',
        'device-alias name PURE_CT0_FC2 pwwn 52:4a:93:77:59:62:b2:02\n',
        'device-alias name PURE_CT1_FC2 pwwn 52:4a:93:77:59:62:b2:12\n',
        'device-alias name Pure_CT1_FC0 pwwn 52:4a:93:77:59:62:b2:10\n',
        'device-alias name CVRPUXSQP01a_QL pwwn 21:01:00:1b:32:3a:f0:a2\n',
        'device-alias name CX480_0303_SPA2 pwwn 50:06:01:62:44:60:27:6f\n',
        'device-alias name CX480_0303_SPA4 pwwn 50:06:01:64:44:60:27:6f\n',
        'device-alias name CX480_0303_SPA6 pwwn 50:06:01:66:44:60:27:6f\n',
        'device-alias name CX480_0303_SPB2 pwwn 50:06:01:6a:44:60:27:6f\n',
        'device-alias name CX480_0303_SPB4 pwwn 50:06:01:6c:44:60:27:6f\n'
    ]
    expected_end = [
        'device-alias name MFXRFPVDIESX01_Virtual_HBA0 pwwn 20:00:00:25:b5:01:a0:1f\n',
        'device-alias name MFXRFPVDIESX02_Virtual_HBA0 pwwn 20:00:00:25:b5:01:a0:0f\n',
        'device-alias name MFXRFPVDIESX03_Virtual_HBA0 pwwn 20:00:00:25:b5:01:a0:3f\n',
        'device-alias name MFXRFPVDIESX04_Virtual_HBA0 pwwn 20:00:00:25:b5:01:a0:2f\n',
        'device-alias name MFXRFPVDIESX05_Virtual_HBA0 pwwn 20:00:00:25:b5:01:a0:5f\n',
        'device-alias name MFXRFPVDIESX06_Virtual_HBA0 pwwn 20:00:00:25:b5:01:a0:4f\n',
        'device-alias name MFXRFPVDIESX07_Virtual_HBA0 pwwn 20:00:00:25:b5:01:a0:7f\n',
        'device-alias name MFXRFPVDIESX08_Virtual_HBA0 pwwn 20:00:00:25:b5:01:a0:6f\n',
        '\n',
        'Total number of entries = 68\n'
    ]
    assert expected_start == result['show_device_alias_database'][:10]
    assert expected_end == result['show_device_alias_database'][-10:]
    assert len(result['show_device_alias_database']) == 70

def test_get_show_flogi_database(support_show_parser):
    """Test that we get show flogi database output from a log."""
    result = support_show_parser.get_fields(['show_flogi_database'])
    expected_start = [
        '--------------------------------------------------------------------------------\n',
        'INTERFACE        VSAN    FCID           PORT NAME               NODE NAME       \n',
        '--------------------------------------------------------------------------------\n',
        'fc1/2            50    0x320300  10:00:00:00:c9:fc:15:c6 20:00:00:00:c9:fc:15:c6\n',
        '                           [MFRPNTPSSSPAPP1_ELX_HBA1]\n',
        'fc1/3            50    0x3202c0  21:00:00:24:ff:53:02:51 20:00:00:24:ff:53:02:51\n',
        '                           [MFRPESXPSS07_QL_HBA0]\n',
        'fc1/4            50    0x320280  21:00:00:24:ff:53:02:05 20:00:00:24:ff:53:02:05\n',
        '                           [MFRPESXPSS06_QL_HBA0]\n',
        'fc1/5            50    0x320220  21:00:00:24:ff:53:01:dd 20:00:00:24:ff:53:01:dd\n'
    ]
    expected_end = [
        'port-channel1    50    0x3200a6  20:01:00:25:b5:a0:a1:3e 20:00:00:25:b5:a0:01:3e\n',
        '                           [MFXRFUCSESX06_Virtual_HBA1]\n',
        'port-channel1    50    0x3200a7  20:01:00:25:b5:a0:a1:1e 20:00:00:25:b5:a0:01:1e\n',
        '                           [MFXRFUCSESX08_Virtual_HBA1]\n',
        'port-channel1    50    0x3200a8  20:01:00:25:b5:a0:a1:0e 20:00:00:25:b5:a0:01:0e\n',
        '                           [MFXRFUCSESX07_Virtual_HBA1]\n',
        'port-channel1    50    0x3200c0  24:01:00:05:73:d2:59:c0 20:32:00:05:73:d2:59:c1\n',
        '\n',
        'Total number of flogi = 38.\n',
        '\n'
    ]
    assert expected_start == result['show_flogi_database'][:10]
    assert expected_end == result['show_flogi_database'][-10:]
    assert len(result['show_flogi_database']) == 80

def test_get_show_interface(support_show_parser):
    """Test that we get show interface information from a log."""
    result = support_show_parser.get_fields(['show_interface'])
    expected_start = [
        'fc1/1 is down (Link failure or not-connected)\n',
        '    Hardware is Fibre Channel, SFP is short wave laser w/o OFC (SN)\n',
        '    Port WWN is 20:01:00:0d:ec:3c:06:00\n',
        '    Admin port mode is F, trunk mode is off\n',
        '    snmp link state traps are enabled\n',
        '    Port vsan is 50\n',
        '    Receive data field Size is 2112\n',
        '    Beacon is turned off\n',
        '    5 minutes input rate 0 bits/sec,0 bytes/sec, 0 frames/sec\n',
        '    5 minutes output rate 0 bits/sec,0 bytes/sec, 0 frames/sec\n'
    ]
    expected_end = [
        '    0 Rx pause\n',
        '  TX\n',
        '    0 unicast packets  0 multicast packets  0 broadcast packets\n',
        '    0 output packets  0 bytes\n',
        '    0 jumbo packets\n',
        '    0 output error  0 collision  0 deferred  0 late collision\n',
        '    0 lost carrier  0 no carrier  0 babble  0 output discard\n',
        '    0 Tx pause\n',
        '\n',
        '\n'
    ]
    assert expected_start == result['show_interface'][:10]
    assert expected_end == result['show_interface'][-10:]
    assert len(result['show_interface']) == 2524

def test_get_show_switchname(support_show_parser):
    """Test that we get show switchname command output from a log."""
    result = support_show_parser.get_fields(['show_switchname'])
    expected = ['MFXRFPEMC9513-A\n']
    assert expected == result['show_switchname']
    assert len(result['show_switchname']) == 1

def test_get_show_zoneset_active(support_show_parser):
    """Test that we get show zoneset active output from a log."""
    result = support_show_parser.get_fields(['show_zoneset_active'])
    expected_start = [
        'zoneset name EVEN_RFP vsan 50\n',
        '  zone name CVRPUXSQQ01a_CT0_FC0 vsan 50\n',
        '  * fcid 0x321800 [pwwn 21:00:00:e0:8b:92:ce:84] [CRPUXSQQ01a_Qlogic_HBA2]\n',
        '  * fcid 0x3201a2 [pwwn 52:4a:93:77:59:62:b2:00] [PURE_CT0_FC0]\n',
        '  \n',
        '  zone name CVRPUXSQQ01a_CT0_FC2 vsan 50\n',
        '  * fcid 0x321800 [pwwn 21:00:00:e0:8b:92:ce:84] [CRPUXSQQ01a_Qlogic_HBA2]\n',
        '  * fcid 0x320202 [pwwn 52:4a:93:77:59:62:b2:02] [PURE_CT0_FC2]\n',
        '  \n',
        '  zone name CVRPUXSQQ01a_CT1_FC0 vsan 50\n'
    ]
    expected_end = [
        '  zone name MFRPNTPSSSPSQP1_CT1_FC0 vsan 50\n',
        '  * fcid 0x320320 [pwwn 10:00:00:00:c9:fc:13:3a] [MFRPNTPSSSPSQP1_ELX_HBA1]\n',
        '  * fcid 0x320701 [pwwn 52:4a:93:77:59:62:b2:10] [Pure_CT1_FC0]\n',
        '  \n',
        '  zone name MFRPNTPSSSPSQP1_CT1_FC2 vsan 50\n',
        '  * fcid 0x320320 [pwwn 10:00:00:00:c9:fc:13:3a] [MFRPNTPSSSPSQP1_ELX_HBA1]\n',
        '  * fcid 0x320721 [pwwn 52:4a:93:77:59:62:b2:12] [PURE_CT1_FC2]\n',
        '  \n',
        '  zone name $default_zone$ vsan 50\n',
        '  * fcid 0x3200c0\n'
    ]
    assert expected_start == result['show_zoneset_active'][:10]
    assert expected_end == result['show_zoneset_active'][-10:]
    assert len(result['show_zoneset_active']) == 563

def test_get_show_version(support_show_parser):
    """Test that we get show version output from a log."""
    result = support_show_parser.get_show_version()
    expected = [
        'Cisco Nexus Operating System (NX-OS) Software\n',
        'TAC support: http://www.cisco.com/tac\n',
        'Documents: http://www.cisco.com/en/US/products/ps9372/tsd_products_support_series_home.html\n',
        'Copyright (c) 2002-2014, Cisco Systems, Inc. All rights reserved.\n',
        'The copyrights to certain works contained herein are owned by\n',
        'other third parties and are used and distributed under license.\n',
        'Some parts of this software are covered under the GNU Public\n',
        'License. A copy of the license is available at\n',
        'http://www.gnu.org/licenses/gpl.html.\n',
        '\n',
        'Software\n',
        '  BIOS:      version 1.0.10\n',
        '  loader:    version N/A\n',
        '  kickstart: version 6.2(9a)\n',
        '  system:    version 6.2(9a)\n',
        '  BIOS compile time:       01/08/09\n',
        '  kickstart image file is: bootflash:///m9500-sf2ek9-kickstart-mz.6.2.9a.bin\n',
        '  kickstart compile time:  8/20/2014 11:00:00 [10/14/2014 06:54:36]\n',
        '  system image file is:    bootflash:///m9500-sf2ek9-mz.6.2.9a.bin\n',
        '  system compile time:     8/20/2014 11:00:00 [10/14/2014 08:24:00]\n',
        '\n',
        '\n',
        'Hardware\n',
        '  cisco MDS 9513 (13 Slot) Chassis ("Supervisor/Fabric-2a")\n',
        '  Motorola, 7447A, altivec  with 2071288 kB of memory.\n',
        '  Processor Board ID JAF1647ATGK\n',
        '\n',
        '  Device name: MFXRFPEMC9513-A\n',
        '  bootflash:    1000944 kB\n',
        '  slot0:              0 kB (expansion flash)\n',
        '\n',
        'Kernel uptime is 688 day(s), 15 hour(s), 15 minute(s), 1 second(s)\n',
        '\n',
        'Last reset \n',
        '  Reason: Unknown\n',
        '  System version: 6.2(3)\n',
        '  Service: \n',
        '\n',
        'plugin\n',
        '  Core Plugin\n'
    ]
    assert result == expected

def test_get_switchname(support_show_parser):
    """Test that we get the switchname properly."""
    expected = 'MFXRFPEMC9513-A'
    assert expected == support_show_parser.get_switchname()

def test_get_uptime(support_show_parser):
    """Test that we get uptime from a log."""
    result = support_show_parser.get_uptime()
    assert result == '688 day(s), 15 hour(s), 15 minute(s), 1 second(s)'

def test_get_version(support_show_parser):
    """Test that we get version from a log."""
    result = support_show_parser.get_version()
    assert result == '6.2(3)'

def test_get_zonesets(support_show_parser):
    """Test that we get zonesets from a log."""
    result = support_show_parser.get_zonesets()
    assert result['EVEN_RFP'].name == 'EVEN_RFP'
    assert len(result['EVEN_RFP'].zones['CVRPUXSQP01a_CT1_FC2'].members) == 2


from flask_restplus import Api, Resource, fields
from werkzeug.contrib.fixers import ProxyFix
from flask import Flask, url_for, jsonify
from elasticsearch import Elasticsearch

es_host = {"host": "elasticsearch1", "port": 9200}
es = Elasticsearch([es_host], retry_on_timeout=True, maxsize=25)

app = Flask(__name__)
api = Api(app,
	      version='1.0',
          title='Get Product And \"like\" Product Details',
          description='Simple Product API Endpoints',
          prefix="/v1",
          contact="john@swarmee.net",
          contact_url="www.swarmee.net"
         )
app.wsgi_app = ProxyFix(app.wsgi_app)
ns = api.namespace('product', description='Simple Product API operations')

@ns.route('/search/<searchTerms>')
class productSearch(Resource):
    def get(self, searchTerms):
        productSearchResponse = es.search(index="products", body="{\"query\": {\"query_string\": {\"query\": \"%s\"}}}" % searchTerms)
        productSearchClientResponse = []
        for row in productSearchResponse["hits"]["hits"]:
          productSearchClientResponse.append(row["_source"])
        return jsonify(productSearchClientResponse)

@ns.route('/<productNumber>')
class productNumber(Resource):
    def get(self, productNumber):
        productNumberSearchResponse = es.search(index="products", body="{\"query\": {\"match\": {\"_id\": \"%s\"}}}" % productNumber)
        productNumberClientResponse = []
        for row in productNumberSearchResponse["hits"]["hits"]:
          productNumberClientResponse.append(row["_source"])
        return jsonify(productNumberClientResponse)

@ns.route('/like/<productNumber>')
class likeProductNumber(Resource):
    def get(self, productNumber):
        likeProductNumberSearchResponse = es.search(index="products", body="{\"query\": {\"match\": {\"_id\": \"%s\"}}}" % productNumber, filter_path=['hits.hits._source.productName'])
        for row in likeProductNumberSearchResponse["hits"]["hits"]:
          likeProductNumberClientResponse = row["_source"]["productName"]
        resp3 = es.search(index="products", body="{\"query\": { \"common\": {\"street\": {\"query\": \"%s\",\"cutoff_frequency\": 0.1}}}}" % likeProductNumberClientResponse, filter_path=['hits.hits._source.*'])
        finalResponse = []
        for row in resp3["hits"]["hits"]:
          finalResponse.append(row["_source"])
        return jsonify(finalResponse)

@ns.route('/count')
class productCount(Resource):
    def get(self):
        resp = es.count(index="products", filter_path=['-took','-timed_out','-_shards'])
        return resp

@ns.route('/health')
class elasticHealth(Resource):
    def get(self):
        resp = es.cluster.health(filter_path=['status'])
        return resp

if __name__ == '__main__':
    app.run(debug=True)


import time

from django.contrib.sites.models import Site
from django.core.paginator import PageNotAnInteger, EmptyPage, Paginator
from django.template.loader import render_to_string
from django.urls import reverse

from tortulka.settings import SEPARATOR
from .encryption_util import encrypt
from .models import Subscriber
from .tasks import send_news_email

def paginate(objects, size, request, context, var_name='object_list'):
    """Paginate objects provided by view.

    This function takes:
    * list of elements;
    * number of objects per page;
    * request object to get url parameters from;
    * context to set new variables into;
    * var_name - variable name for list of objects.

    It returns updated context object.
    """

    # apply pagination
    paginator = Paginator(objects, size)

    # try to get page number from request
    page = request.GET.get('page', 1)
    try:
        object_list = paginator.page(page)
    except PageNotAnInteger:
        # if page is not an integer, deliver first page
        object_list = paginator.page(1)
    except EmptyPage:
        # if page is out of range (e.g. 9999),
        # deliver last page of results
        object_list = paginator.page(paginator.num_pages)

    # set variables into context
    context[var_name] = object_list
    context['is_paginated'] = object_list.has_other_pages()
    context['page_obj'] = object_list
    context['paginator'] = paginator

    return context

def filtering(request, object_list):
    """Filter objects provided by view.

    This function takes:
    * request:
    * list of elements;

    It returns filtered Queryset.
    """

    # searching
    search_product = request.GET.get('q', '')
    if search_product:
        object_list = object_list.filter(
            title__icontains=search_product)

    # filter by category
    category_filter = request.GET.get('category', '')
    if category_filter == 'all':
        object_list = object_list
    elif category_filter:
        object_list = object_list.filter(category__slug=category_filter)

    # filter by tiers
    tiers_filter = request.GET.get('tiers', '')
    if tiers_filter == 'all':
        object_list = object_list
    elif tiers_filter:
        object_list = object_list.filter(category__slug=tiers_filter)

    # ordering
    order_by = request.GET.get('order_by', '')
    if order_by == 'newest':
        object_list = object_list.order_by('-added', 'id')
    elif order_by == 'oldest':
        object_list = object_list.order_by('added', 'id')
    elif order_by == 'abc':
        object_list = object_list.order_by('title', 'id')

    return object_list

def send_newsletter(subject, message):
    """
    Send letter to newsletter subscribers
    :param subject: str - subject of letter
    :param message:  str - message of letter
    :return: None
    """
    subscribers = list(Subscriber.objects.values_list('email', flat=True))
    for subscriber in subscribers:
        token = encrypt(subscriber + SEPARATOR + str(time.time()))
        url = reverse('unsubscribe')
        domain = Site.objects.get_current().domain
        confirmation_url = domain + url + '?token=' + token
        message = render_to_string('news_letter.html', {
            'message': message,
            'confirmation_url': confirmation_url
        })
        send_news_email.delay(subject, message, subscriber)

def send_subscription_email(email, url):
    """
    Send email to subscriber for confirmation subscription

    :param email: Subscriber email
    :param url: url to subscription confirmation page
    :return: None
    """
    token = encrypt(email + SEPARATOR + str(time.time()))
    confirmation_url = url + '?token=' + token
    mail_subject = 'Підтвердження підписки на розсилку'
    message = render_to_string('subscription.html', {
        'confirmation_url': confirmation_url
    })
    to_email = email
    send_news_email.delay(mail_subject, message, to_email)


# coding: utf-8

import cv2 as cv
import numpy as np
import math as m
from omrtemplate import get_template_rois
from pyzbar.pyzbar import decode
import re

def get_testset_answers(filename):
    question_dictionary_re = re.compile(r"^\s*questiondictionary")
    question_variants = []
    randans_dictionary_re = re.compile(r"^\s*randomizedanswersdictionary")
    answers_variants = []
    corr_dictonary_re = re.compile(r"^\s*correctiondictionary")
    answer_scores = []
    r_c_func_re = re.compile(r"c\(\s*(\w+\s*(,\s*\w+\s*)+)\s*\)")
    glob_list_container_re = re.compile(r"list\(\s*(.*)\s*\)")
    list_container_re = re.compile(r"list\(\s*(c\(\s*\w+\s*(,\s*\w+\s*)*\s*\)\s*(,\s*c\(\s*\w+\s*(,\s*\w+\s*)*\s*\))*)\s*\)")

    r_script = open(filename, "r")
    line = r_script.readline()
    while line:
        if re.match(question_dictionary_re, line) or \
                re.match(randans_dictionary_re, line) or \
                re.match(corr_dictonary_re, line):
            start_line = line.strip()
            data = start_line
            balance = data.count('(') - data.count(')')
            line = r_script.readline()
            while balance != 0 and line:
                data += line.strip()
                balance = data.count('(') - data.count(')')
                line = r_script.readline()
            content_match = re.search(glob_list_container_re, data)
            if content_match:
                list_content = content_match.group(1)
                if re.match(question_dictionary_re, start_line):
                    for p in re.finditer(r_c_func_re, list_content):
                        question_variants.append((*(int(x.strip()) for x in p.group(1).split(",")),))
                elif re.match(corr_dictonary_re, start_line):
                    for p in re.finditer(r_c_func_re, list_content):
                        answer_scores.append((*(float(x.strip()) for x in p.group(1).split(",")),))
                elif re.match(randans_dictionary_re, start_line):
                    for p in re.finditer(list_container_re, list_content):
                        answers = []
                        for q in re.finditer(r_c_func_re, p.group(1)):
                            answers.append((*(int(x.strip()) for x in q.group(1).split(",")),))
                        answers_variants.append(answers)
        line = r_script.readline()

    return question_variants, answers_variants, answer_scores

def process_answer_sheet(answer_img,
                         template_filename,
                         contour_eps=0.02,
                         compactness_eps=0.1,
                         area_eps=0.7,
                         closeness_eps=1.5,
                         clahe_cliplimit=4.0,
                         blur_ksize=3,
                         laplacian_ksize=5,
                         skip_adaptive_thresh=True,
                         thresh_block_size=33,
                         thresh_bias=3):
    # Convert to gray tones
    img_gray = cv.cvtColor(answer_img, cv.COLOR_BGR2GRAY)

    # Decode the QR-code
    qr_code = decode(img_gray)[0]
    qrdata = qr_code.data.decode('utf-8').split()
    department_id = qrdata[0]
    course_id = qrdata[1]
    exam_date = qrdata[2]
    variants_count = int(qrdata[3])
    questions_count = int(qrdata[4])
    mx = 0
    my = 0
    for p in qr_code.polygon:
        mx = mx + p.x
        my = my + p.y

    mx = mx / 4
    my = my / 4

    height = img_gray.shape[0]
    width = img_gray.shape[1]
    img_corners = {(0, 0): 1, (width, 0): 0, (0, height): 2, (width, height): 3}
    min_dist = m.sqrt(height ** 2 + width ** 2)
    rotation_count = 0
    for k in img_corners:
        dist = m.sqrt((mx - k[0]) ** 2 + (my - k[1]) ** 2)
        if min_dist > dist:
            min_dist = dist
            rotation_count = img_corners[k]

    # Rotate the image such that the page is correctly oriented
    for c in range(rotation_count):
        img_gray = cv.transpose(img_gray)
        img_gray = cv.flip(img_gray, flipCode=1)

    qr_code = decode(img_gray)[0]
    qr_vertices = np.zeros((4, 1, 2), dtype=np.int32)
    for idx in range(4):
        p = qr_code.polygon[idx]
        qr_vertices[idx][0][0] = p.x
        qr_vertices[idx][0][1] = p.y

    # get template informations from the svg file
    page_dims, qr_dims, bounding_rect, answer_bubbles = get_template_rois(template_filename)

    # crop the page to edges, in relation to qr shape
    qr_norms = [np.linalg.norm(d) for d in qr_vertices]
    qr_vertices = np.roll(qr_vertices, -np.argmin(qr_norms), axis=0)
    qr_deltas = qr_vertices - np.roll(qr_vertices, 1, axis=0)
    qr_lengths = [int(np.linalg.norm(d)) for d in qr_deltas]
    new_qr_width = max(qr_lengths)
    new_qr_height = int(qr_dims['height'] * new_qr_width / qr_dims['width'])
    qr_x = qr_vertices[0][0][0]
    qr_y = qr_vertices[0][0][1]
    new_qr_vertices = np.array([
        [qr_x, qr_y],
        [qr_x, qr_y + new_qr_height - 1],
        [qr_x + new_qr_width - 1, qr_y + new_qr_height - 1],
        [qr_x + new_qr_width - 1, qr_y]], dtype="float32")
    reshaped_qr_vertices = np.float32(np.reshape(qr_vertices, (4, 2)))
    # compute the perspective transform matrix and then apply it
    perspective_tm = cv.getPerspectiveTransform(reshaped_qr_vertices, new_qr_vertices)
    img_qr_box = cv.warpPerspective(img_gray,
                                    perspective_tm,
                                    (int(img_gray.shape[1] * 1.1), int(img_gray.shape[0] * 1.1)))
    marg_lx = bounding_rect['x'] / 3
    marg_ly = bounding_rect['y'] / 3
    # marg_rx = (page_dims['width'] - (bounding_rect['x'] + bounding_rect['width'])) / 3
    # marg_ry = (page_dims['height'] - (bounding_rect['y'] + bounding_rect['height'])) / 3
    new_page_xul = int(new_qr_vertices[0][0] - (qr_dims['x'] + marg_lx) * new_qr_width / qr_dims['width'] * 1.2)
    if new_page_xul < 0:
        new_page_xul = 0
    new_page_yul = int(new_qr_vertices[0][1] - (qr_dims['y'] + marg_ly) * new_qr_height / qr_dims['height'])
    if new_page_yul < 0:
        new_page_yul = 0
    new_page_xlr = int(new_qr_vertices[0][0] +
                       (page_dims['width'] - qr_dims['x']) * new_qr_width / qr_dims['width'])
    if new_page_xlr > img_qr_box.shape[1]:
        new_page_xlr = img_qr_box.shape[1]
    new_page_ylr = int(new_qr_vertices[0][1] +
                       (page_dims['height'] - qr_dims['y']) * new_qr_height / qr_dims['height'] * 1.2)
    if new_page_ylr > img_qr_box.shape[0]:
        new_page_ylr = img_qr_box.shape[0]

    # crop the image
    img_qr_box = img_qr_box[new_page_yul:new_page_ylr, new_page_xul:new_page_xlr]

    # create a CLAHE object
    clahe = cv.createCLAHE(clipLimit=clahe_cliplimit)
    # apply CLAHE for better contrast
    img_clahe = clahe.apply(img_qr_box)

    # apply blur
    img_blur = cv.GaussianBlur(img_clahe, (blur_ksize, blur_ksize), 0)

    if not skip_adaptive_thresh:
        # apply adaptive thresholding
        img_th = cv.adaptiveThreshold(img_blur, 255, cv.ADAPTIVE_THRESH_GAUSSIAN_C,
                                      cv.THRESH_BINARY, thresh_block_size, thresh_bias)

    # apply laplace edge detection
    img_laplace = cv.Laplacian(img_th if not skip_adaptive_thresh else img_blur, cv.CV_8U, ksize=laplacian_ksize)

    # find contours in the edged image
    _, contours, _ = cv.findContours(img_laplace, cv.RETR_LIST, cv.CHAIN_APPROX_SIMPLE)

    # select the max area contour
    contour = max(contours, key=cv.contourArea)
    perimeter = cv.arcLength(contour, True)
    approx_contour = cv.approxPolyDP(contour, contour_eps * perimeter, True)
    vertices = cv.convexHull(approx_contour, clockwise=True)

    box_frac = bounding_rect['stroke-width'] / bounding_rect['width']
    norms = [np.linalg.norm(d) for d in vertices]
    vertices = np.roll(vertices, -np.argmin(norms), axis=0)
    deltas = vertices - np.roll(vertices, 1, axis=0)
    lengths = [int(np.linalg.norm(d)) for d in deltas]
    dim = max(lengths)
    new_height = dim
    new_width = dim * int(bounding_rect['width'] - bounding_rect['stroke-width']) // int(
        bounding_rect['height'] - bounding_rect['stroke-width'])
    new_vertices = np.array([
        [0, 0],
        [0, new_height - 1],
        [new_width - 1, new_height - 1],
        [new_width - 1, 0]], dtype="float32")
    vertices = np.float32(np.reshape(vertices, (4, 2)))
    # compute the perspective transform matrix and then apply it
    M = cv.getPerspectiveTransform(vertices, new_vertices)
    img_box = cv.warpPerspective(img_qr_box, M, (new_width, new_height))
    crop_width = int(np.round(img_box.shape[0] * box_frac))

    translate_tm = np.identity(3)
    translate_tm[0, 2] = -bounding_rect['x']
    translate_tm[1, 2] = -bounding_rect['y']
    scale_tm = np.identity(3)
    scale_factor = img_box.shape[1] / (bounding_rect['width'] + bounding_rect['stroke-width'])
    scale_tm[0, 0] = scale_factor
    scale_tm[1, 1] = scale_factor
    map_tm = scale_tm @ translate_tm

    # map bubbles template coordinates to image coordinates
    initial_r = np.array([answer_bubbles[0]['r'], 0, 1.0], ndmin=2).T
    mapped_r = np.ceil(scale_tm @ initial_r)
    circle_r = mapped_r[0][0]
    circle_area = m.pi * circle_r ** 2
    for idx in range(len(answer_bubbles)):
        initial_coords = np.array([answer_bubbles[idx]['cx'], answer_bubbles[idx]['cy'], 1.0], ndmin=2).T
        mapped_coords = map_tm @ initial_coords
        answer_bubbles[idx]['cx'] = int(mapped_coords[0, 0])
        answer_bubbles[idx]['cy'] = int(mapped_coords[1, 0])
        answer_bubbles[idx]['r'] = circle_r

    # crop the margins
    img_box = img_box[crop_width:img_box.shape[0] - crop_width, crop_width:img_box.shape[1] - crop_width]

    # create a CLAHE object
    clahe_box = cv.createCLAHE(clipLimit=clahe_cliplimit)
    # apply CLAHE for better contrast
    img_box_clahe = clahe_box.apply(img_box)

    # apply blur
    img_box_blur = cv.GaussianBlur(img_box_clahe, (blur_ksize, blur_ksize), 0)

    if not skip_adaptive_thresh:
        # apply adaptive thresholding
        img_box_th = cv.adaptiveThreshold(img_box_blur, 255, cv.ADAPTIVE_THRESH_GAUSSIAN_C,
                                          cv.THRESH_BINARY, thresh_block_size, 1)

    # apply canny edge detection
    img_box_canny = cv.Canny(img_box_th if not skip_adaptive_thresh else img_box_blur, 75, 200)

    # find contours in the boxed image
    _, box_contours, _ = cv.findContours(img_box_canny, cv.RETR_LIST, cv.CHAIN_APPROX_SIMPLE)

    by_compactness_contours = []
    for c in box_contours:
        moments = cv.moments(c)
        if moments['mu20'] + moments['mu02'] != 0:
            compactness = moments['m00'] ** 2 / (2 * m.pi * (moments['mu20'] + moments['mu02']))
            if abs(compactness - 1) < compactness_eps:
                by_compactness_contours.append(c)

    by_area_contours = [c for c in by_compactness_contours
                        if abs((circle_area - cv.contourArea(c)) / circle_area) <= area_eps]

    # filter contours by selecting only the contours in expected locations
    # (i.e. by closeness to the expected locations)
    # if several contours are close to the expected locations,
    # only select the largest ones

    for c in by_area_contours:
        moments = cv.moments(c)
        mx = moments['m10'] / moments['m00']
        my = moments['m01'] / moments['m00']
        min_dist = circle_r * (1 + closeness_eps)
        bubble_pair = 0
        for bubble_idx in range(len(answer_bubbles)):
            bubble = answer_bubbles[bubble_idx]
            cx = bubble['cx']
            cy = bubble['cy']
            if min_dist > m.sqrt((cx - mx) ** 2 + (cy - my) ** 2):
                min_dist = m.sqrt((cx - mx) ** 2 + (cy - my) ** 2)
                bubble_pair = bubble_idx
        if abs(min_dist - circle_r) / circle_r < closeness_eps:
            if 'contours' in answer_bubbles[bubble_pair]:
                answer_bubbles[bubble_pair]['contours'].append(c)
            else:
                answer_bubbles[bubble_pair]['contours'] = [c]

    for bubble in answer_bubbles:
        if 'contours' in bubble:
            max_contour = max(bubble['contours'], key=cv.contourArea)
            moments = cv.moments(max_contour)
            mx = int(moments['m10'] / moments['m00'])
            my = int(moments['m01'] / moments['m00'])
            bubble['mx'] = mx
            bubble['my'] = my
            bubble['optimal_contour'] = max_contour

    img_box_result = cv.cvtColor(img_box, cv.COLOR_GRAY2BGR)

    # draw bubbles
    for bubble in answer_bubbles:
        if 'optimal_contour' in bubble:
            cv.circle(img_box_result,
                      (bubble['mx'], bubble['my']), int(circle_r), (0, 255, 255), 1)

    # classify bubbles into 'filled' and 'not filled'
    mask = np.zeros((*img_box_th.shape,) if not skip_adaptive_thresh else (*img_box_blur.shape,), np.uint8)
    bubble_means = []
    for bubble in answer_bubbles:
        if 'optimal_contour' in bubble:
            mask[...] = 0
            cv.circle(mask, (bubble['mx'], bubble['my']), int(circle_r), 255, -1)
            bubble['mean'] = cv.mean(img_box_th if not skip_adaptive_thresh else img_box_blur, mask)[0]
            bubble_means.append(bubble['mean'])
    filled_threshold = np.mean(bubble_means) * 0.9
    for bubble in answer_bubbles:
        if 'optimal_contour' in bubble:
            bubble['is_filled'] = bubble['mean'] < filled_threshold

    # draw bubbles according to the previous classification
    font = cv.FONT_HERSHEY_PLAIN
    font_size = 0.7 * img_box.shape[0] / 1000.0
    bubble_color = (0, 255, 255)
    img_box_result = cv.cvtColor(img_box, cv.COLOR_GRAY2BGR)
    cv.rectangle(img_box_result, (0, 0), (img_box_result.shape[1], img_box_result.shape[0]), (0, 0, 0), 3)

    for bubble in answer_bubbles:
        if 'optimal_contour' in bubble:
            if bubble['is_filled']:
                cv.circle(img_box_result,
                          (bubble['mx'], bubble['my']), int(circle_r), (0, 0, 0), -1)
            cv.circle(img_box_result,
                      (bubble['mx'], bubble['my']), int(circle_r), bubble_color, 1)
            id_parts = bubble['id'].split('-')
            if len(id_parts) > 1:
                text = id_parts[1]
            else:
                text = id_parts[0]
            text = text.upper()
            # get boundary of this text
            text_size = cv.getTextSize(text, font, font_size, 1)
            # get coords based on boundary
            text_x = bubble['mx'] - text_size[0][0] // 2
            text_y = bubble['my'] + text_size[0][1] // 2
            cv.putText(img_box_result, text,
                       (text_x, text_y),
                       font, font_size,
                       (255, 255, 255) if bubble['is_filled'] else (0, 0, 0), 1, cv.LINE_AA)

    # find all answers
    answers = {}
    for bubble in answer_bubbles:
        if 'optimal_contour' in bubble:
            id, pos = bubble['id'].split('-')
            if id in answers:
                answers[id][pos] = bubble['is_filled']
            else:
                answers[id] = {pos: bubble['is_filled']}

    # select only valid answers
    valid_answers = {}
    for key in answers:
        try:
            q = int(key)
            if (q <= questions_count and len(answers[key]) == 4 and
                    list(answers[key].values()).count(True) == 1):
                valid_answers[q] = list(answers[key].keys())[list(answers[key].values()).index(True)]
        except ValueError:
            pass

    # get variant number
    variant = 0
    if ('v' in answers and len(answers['v']) == variants_count and
            list(answers['v'].values()).count(True) == 1):
        variant = int(list(answers['v'].keys())[list(answers['v'].values()).index(True)])

    return img_box_result, qr_code.data.decode('utf-8'), variant, valid_answers


import sys
sys.path.append("../")
from sklearn.neighbors import KNeighborsClassifier
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.metrics import classification_report
from pyimagesearch.preprocessing.simplepreprocessor import SimplePreprocessor
from pyimagesearch.datasets.simpledatasetloader import SimpleDatasetLoader
from imutils import paths
import argparse

# construct the argument parse and parse the arguments
ap = argparse.ArgumentParser()
ap.add_argument("-d", "--dataset", required=True, help="path to input dataset")
ap.add_argument("-k", "--neighbors", type=int, default=1, help="# nearest neighbors for classification")
ap.add_argument("-j", "--jobs", type=int, default=-1, help="# of jobs for k-NN distance (-1 uses all available cores)")

args = vars(ap.parse_args())

#List of images
imagePaths = list(paths.list_images(args["dataset"]))

# initialize the image preprocessor, load the dataset from disk, and reshape the data matrix
sp = SimplePreprocessor(32, 32)
sdl = SimpleDatasetLoader(preprocessors=[sp])

print("[INFO] loading data......")
(data, labels) = sdl.load(imagePaths, verbose=250)
print("[INFO] data shape {}".format(data.shape))
data = data.reshape((data.shape[0], 3072))

#memory taken by images
print("[INFO] features matrix: {:.1f}MB".format(data.nbytes / (1024 * 1000.0)))

#Encoding the labes as integers
le = LabelEncoder()
labels = le.fit_transform(labels)

#partition the data into train and test set 75% as train and rest as test
(trainX, testX, trainY, testY) = train_test_split(data, labels, test_size=0.25, random_state=42)

#training and evaluatin knn classifier
print("[INFO] evaluating knn classifier....")

model = KNeighborsClassifier(n_neighbors=args['neighbors'], n_jobs=args["jobs"])
model.fit(trainX, trainY)
print(classification_report(testY, model.predict(testX), target_names=le.classes_))


import io
import os

import pytest

from dynaconf import default_settings
from dynaconf import LazySettings
from dynaconf.loaders.py_loader import load
from dynaconf.loaders.py_loader import try_to_load_from_py_module_name
from dynaconf.utils import DynaconfDict

def test_py_loader_from_file(tmpdir):

    settings = DynaconfDict()
    dummy_path = tmpdir.join("dummy_module.py")
    with io.open(
        str(dummy_path), "w", encoding=default_settings.ENCODING_FOR_DYNACONF
    ) as f:
        f.write('FOO = "bar"')

    load(settings, "dummy_module.py")
    os.remove("dummy_module.py")
    load(settings, "dummy_module.py")  # will be ignored not found

    assert settings.get("FOO") == "bar"

def test_py_loader_from_module(tmpdir):

    settings = DynaconfDict()
    dummy_folder = tmpdir.mkdir("dummy")

    dummy_folder.join("dummy_module.py").write('FOO = "bar"')
    dummy_folder.join("__init__.py").write('print("initing dummy...")')

    load(settings, "dummy.dummy_module")

    assert settings.exists("FOO")

def test_try_to_load_from_py_module_name(tmpdir):
    settings = DynaconfDict()
    dummy_folder = tmpdir.mkdir("dummy")

    dummy_folder.join("dummy_module.py").write('FOO = "bar"')
    dummy_folder.join("__init__.py").write('print("initing dummy...")')

    try_to_load_from_py_module_name(settings, "dummy.dummy_module")

    assert settings.exists("FOO")

def test_negative_try_to_load_from_py_module_name(tmpdir):
    settings = DynaconfDict()
    with pytest.raises(ImportError):
        try_to_load_from_py_module_name(settings, "foo.bar.dummy")

def test_silently_try_to_load_from_py_module_name(tmpdir):
    settings = DynaconfDict()
    try_to_load_from_py_module_name(settings, "foo.bar.dummy", silent=True)

    assert settings.exists("FOO") is False

def test_py_loader_from_file_dunder(clean_env, tmpdir):
    """Test load with dunder settings"""

    settings = LazySettings(
        DATABASES={
            "default": {
                "NAME": "db",
                "ENGINE": "module.foo.engine",
                "ARGS": {"timeout": 30},
                "PORTS": [123, 456],
            }
        }
    )
    dummy_path = tmpdir.join("dummy_module.py")
    with io.open(
        str(dummy_path), "w", encoding=default_settings.ENCODING_FOR_DYNACONF
    ) as f:
        f.write('F = "bar"')
        f.write("\n")
        f.write('COLORS__white__code = "#FFFFFF"')
        f.write("\n")
        f.write('DATABASES__default__ENGINE = "other.module"')

    load(settings, "dummy_module.py")
    os.remove("dummy_module.py")
    load(settings, "dummy_module.py")  # will be ignored not found

    assert settings.get("F") == "bar"
    assert settings.COLORS == {"white": {"code": "#FFFFFF"}}
    assert settings.DATABASES.default.NAME == "db"
    assert settings.DATABASES.default.ENGINE == "other.module"