#!/usr/bin/env python
"""This program offers the user random articles from Wikipedia to read. On
selection of an articles, the program opens the article in a new browser tab.
The user can set how many articles to show at a time (from 1 article to 10).

This program only runs on Python v3.0 or higher.
"""
from getopt import getopt, error
from json import loads
from webbrowser import open_new_tab
import sys

__author__ = "Jeffrey Chan"
__version__ = "1.0.0"

if sys.version_info[0] >= 3:
    from urllib.request import urlopen

"""This is the URL to the Wikipedia API to get 10 random articles."""
wikipedia_random_url = ("http://en.wikipedia.org/w/api.php?action=query&"
                        "list=random&rnnamespace=0&format=json&rawcontinue&"
                        "rnlimit=10")

"""This is the URL to any article on Wikipedia - once the ID is added on."""
wikipedia_article_url = "http://en.wikipedia.org/wiki?curid="

"""Help message for the user."""
help_message = ("This program offers the user random articles from Wikipedia "
                "to read. Articles will be opened in a new browser tab.\n"
                "Use -n or --number= to set the number of articles to display "
                "at any one time (from 1 to 10).")

"""A queue of dictionaries representing Wikipedia articles to eventually
display to the user.
"""
queue = []

"""Current amount of articles to show to the user."""
articles_to_show = 5

def call_Wikipedia():
    """Gets random articles from Wikipedia (10 at a time) and stores them in queue.
    """
    global queue
    try:
        data = urlopen(wikipedia_random_url)
        data_dict = loads(data.read().decode("utf-8"))
        data.close()
        articles = data_dict["query"]["random"]
        queue += articles
        return True
    except Exception as e:
        print(e)
        return False

def offerArticles():
    """Displays articles to the user. Calls call_Wikipedia to fill up queue when
    necessary. Offers user option to select an article by number (will open
    article in new browser tab), get new articles choices, or exit the program.

    This is the main function of the program and is constantly running once the
    program is started waiting for the user to give commands - hence the while True
    loop.
    """
    global queue
    while True:
        try:
            if not queue or len(queue) < articles_to_show:
                if not call_Wikipedia():
                    print("Unable to fetch Articles now")
                    return 1
            current =  queue[:articles_to_show]
            queue = queue[articles_to_show:]
            for i in range(articles_to_show):
                print("[{:d}] {:s}".format(i, current[i]["title"]))
            user_input = input("Select an article by number, press enter to "
                               "get new articles, or type exit.\n")
            while user_input:
                if appropriate_number(user_input, 0, articles_to_show):
                    user_choice = int(user_input)
                    open_new_tab(wikipedia_article_url +
                                 str(current[user_choice]["id"]))
                elif user_input == "exit":
                    return 0
                else:
                    print("That's not a valid option.")
                user_input = input("Select another article by number, "
                                   "press enter to get new articles, or "
                                   "type exit.\n")
        except KeyboardInterrupt:
            print("\n")
            return 0

def appropriate_number(user_input, lower_limit=1, upper_limit=10):
    """Returns true if user input is a number that is bigger than or equal to
    the lower limit and smaller than the upper limit. Returns false otherwise.
    """
    try:
        a = int(user_input)
        if a < lower_limit:
            return False
        elif a >= upper_limit:
            return False
        else:
            return True
    except ValueError:
        return False

def main(*argv):
    """This function starts the program. Provides help messages when the user asks
    for one as well as when user uses program inappropriately. Allows user to
    set the amount of articles to display at one time.
    """
    global articles_to_show
    try:
        if sys.version_info[0] < 3:
            print("Please use Python 3.0 or above.")
            return 1
        opts, args = getopt(argv[0][1:],"hn:",["help", "number="])
    except error as msg:
        print(msg)
        print("for help use --help")
        return 2
    if args and not opts:
        print(help_message)
        return 2
    else:
        for opt, arg in opts:
            if opt in ("-h", "--help"):
                print(help_message)
                return 2
            elif opt in ("-n", "--number"):
                if appropriate_number(arg, upper_limit=11):
                    articles_to_show = int(arg)
                else:
                    print("Please enter a number between 1-10.")
                    return 2
            else:
                print(help_message)
                return 2
    return offerArticles()

"""Calls the function to start the program on run. Exits the program after
running is complete either due to user exiting or an error.
"""
if __name__ == "__main__":
    sys.exit(main(sys.argv))


import findspark
import os
import sys
findspark.init()
spark_home = findspark.find()

#spark_home = os.environ.get('SPARK_HOME', None)
sys.path.insert(0, spark_home + "/python")

# Add the py4j to the path.
# You may need to change the version number to match your install
sys.path.insert(0, os.path.join(spark_home, 'python/lib/py4j-0.8.2.1-src.zip'))

# Adding the library to mysql connector
packages = "mysql:mysql-connector-java:5.1.37"
os.environ["PYSPARK_SUBMIT_ARGS"] = "--packages {0} pyspark-shell".format(
    packages
)

# Initialize PySpark to predefine the SparkContext variable 'sc'
execfile(os.path.join(spark_home, 'python/pyspark/shell.py'))


import os
import argparse
import pickle

import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
import cv2

# This is a hack to find the 'common' folder, for some reason my PYTHONPATH is not working properly
import sys
sys.path.append(os.path.abspath('..'))
from common.image_processing import Cluster, get_galaxy_pixels

labeled_data_dir = os.path.join('..', 'data', 'labeled')
labels_dir = os.path.join('..', 'data', 'labeled.csv')
labeled_data_out_dir = os.path.join('..', 'data', 'labeled_patches')
labels_out_csv = os.path.join('..', 'data', 'labeled_patches.csv')

scored_data_dir = os.path.join('..', 'data', 'scored')
scores_dir = os.path.join('..', 'data', 'scored.csv')
scored_data_out_dir = os.path.join('..', 'data', 'scored_patches')
scored_out_csv = os.path.join('..', 'data', 'scored_patches.csv')

def extract_patches_from_images(df,
                                image_dir,
                                image_out_dir,
                                csv_out_dir,
                                labeled,
                                patch_size=32,
                                background_threshold=5):
    # list of dicts
    image_ids = df.index.values
    patch_info = []
    print("Saving images to: {}".format(image_out_dir))
    for ind, image_id in enumerate(tqdm(image_ids)):
        # if ind == 100:
        #     break
        image_path = os.path.join(image_dir, "{}.png".format(image_id))
        img = cv2.imread(image_path, cv2.IMREAD_GRAYSCALE)

        galaxy_pixels, _ = get_galaxy_pixels(img, threshold=background_threshold)
        clusters = Cluster.find_clusters(img, galaxy_pixels)

        for cluster_ind, cluster in enumerate(clusters):
            patch = cluster.crop_patch(patch_size)

            # patch goes outside the frame of the image, just ignore it
            if patch is None:
                continue

            patch_id = "{}_{:0>2d}.png".format(image_id, cluster_ind)
            patch_path = os.path.join(image_out_dir, patch_id)
            plt.imsave(patch_path, patch, cmap='gray', vmin=0, vmax=255)

            curr_patch_info = {'patch_id': patch_id,
                               'intensity': cluster.get_intensity(),
                               'size': cluster.size(),
                               'score': -1 if labeled else df.at[image_id, 'Actual'],
                               'center': cluster.get_center_pixel()}
            patch_info.append(curr_patch_info)

    patch_info_df = to_df(patch_info)
    print("Saving csv to: {}".format(csv_out_dir))
    patch_info_df.to_csv(csv_out_dir)

def scored_images_galaxy_patches(background_threshold=10,
                                 patch_size=32,
                                 min_galaxy_size=None,
                                 min_score=2):

    scores_df = pd.read_csv(scores_dir, index_col='Id')
    scores_df.index = scores_df.index.astype(str)
    scores_df = scores_df[scores_df.Actual >= min_score]

    if not os.path.exists(scored_data_out_dir):
        os.makedirs(scored_data_out_dir)

    extract_patches_from_images(scores_df,
                                scored_data_dir,
                                scored_data_out_dir,
                                scored_out_csv,
                                False,
                                patch_size=patch_size,
                                background_threshold=background_threshold)

def labeled_images_galaxy_patches(background_threshold=10,
                                  patch_size=32,
                                  min_galaxy_size=None):

    labels_df = pd.read_csv(labels_dir, index_col='Id')
    labels_df.index = labels_df.index.astype(str)
    labels_df = labels_df[labels_df.Actual == 1.0]
    # mark labeled as -1 and store -1 in csv score
    labels_df.Actual = -1

    if not os.path.exists(labeled_data_out_dir):
        os.makedirs(labeled_data_out_dir)

    extract_patches_from_images(labels_df,
                                labeled_data_dir,
                                labeled_data_out_dir,
                                labels_out_csv,
                                True,
                                patch_size=patch_size,
                                background_threshold=background_threshold)

def to_df(data):
    df = pd.DataFrame(data)
    df.set_index('patch_id', inplace=True)

    return df

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Finds the galaxies in the images, and stores the cropped galaxies ('
                                                 'patches) as independent images.')
    parser.add_argument('--background_threshold', type=int, default=5, help='minimum pixel intensity (0-255) for a '
                                                                             'pixel to be considered as part of a galaxy')
    parser.add_argument('--patch_size', type=int, default=32, help='The size of the extracted galaxy patches')
    parser.add_argument('--min_galaxy_size', type=int, default=None, help='disregard galaxies with number of pixels smaller than this number')
    parser.add_argument('--min_score', type=float, default=2, help='disregard fake images i.e images with score < min_score,'
                                                                      'used only if dataset is scored')
    args = parser.parse_args()

    print("Config: ", args)
    print("Working on labeled data:")
    labeled_images_galaxy_patches(background_threshold=args.background_threshold,
                                  patch_size=args.patch_size,
                                  min_galaxy_size=args.min_galaxy_size)

    print("Working on scored data:")
    scored_images_galaxy_patches(background_threshold=args.background_threshold,
                                  patch_size=args.patch_size,
                                  min_galaxy_size=args.min_galaxy_size,
                                  min_score=args.min_score)


import numpy as np
import cv2 as cv
import matplotlib.pyplot as plt
import random

img_name = "Cover_2.png"
encrypted_img_name = "Encrypted_" + img_name

def getKey():
	f = open("key_color.txt", "r")
	k = f.read()
	f.close()
	return k

def decrypt(img, key):
	l = img.shape[0]
	r = img.shape[1]
	klen = len(key)

	tot_msg = "";
	for i in range(l):
		for j in range(r):
			x = key[(i*r + j)%klen]
			x = int(x)
			if((x^img[i][j][0])%2 == 1):
				tot_msg = tot_msg + str(img[i][j][1] & np.uint64(1))
			else:
				tot_msg = tot_msg + str(img[i][j][2] & np.uint64(1))

	mlen = tot_msg[:16]
	mlen = "0b" + mlen
	mlen = int(mlen, 2)
	#print(mlen)
	msg = tot_msg[16:]
	msg = msg[:mlen]
	#print(msg, len(msg))
	val = len(msg)//8
	chunks, chunk_size = len(msg), len(msg)//val
	new_string = [ msg[i:i+chunk_size] for i in range(0, chunks, chunk_size) ]
	dec_msg = ''
	for i in new_string:
		#print(i)
		dec_msg += chr(int(i, 2))
	return dec_msg

if __name__ == "__main__":

	print("Using image " + encrypted_img_name)

	img =  cv.imread(encrypted_img_name)
	#print(img)
	#print(img.shape)

	key = getKey()

	msg = decrypt(img, key)

	print("The decrypted message is :", msg)


from flask import Blueprint, abort, jsonify, request

from sqlalchemy.orm.exc import NoResultFound

from werkzeug.exceptions import BadRequest

from mushi.core.auth import require_auth_token
from mushi.core.db import db_session
from mushi.core.db.models import Tag
from mushi.core.utils.http import jsonify_list

from .exc import ApiError
from .filters import parse_filters

bp = Blueprint('tags', __name__)

@bp.route('/tags/')
@require_auth_token
def list_tags(auth_token):
    query = db_session.query(Tag)

    filters_string = request.args.get('filters')
    if filters_string:
        query = parse_filters(query, Tag, filters_string, [Tag.name])

    count_only = ('count' in request.args) and (request.args['count'] in ('', '1', 'true'))

    if count_only:
        return jsonify({'count': query.count()})
    else:
        limit = request.args.get('limit', 20)
        offset = request.args.get('offset', 0)
        query = query.limit(limit).offset(offset)

        rv = [m.to_dict(max_depth=2) for m in query]
        return jsonify_list(rv)

@bp.route('/tags/', methods=['POST'])
@require_auth_token
def create_tag(auth_token):
    try:
        post_data = request.get_json(force=True)
    except BadRequest as e:
        raise ApiError(e.description)

    new_tag = Tag()
    new_tag.update(post_data)

    db_session.add(new_tag)
    db_session.commit()

    return jsonify(new_tag.to_dict(max_depth=2))

@bp.route('/tags/<name>')
@require_auth_token
def show_tag(auth_token, name):
    try:
        tag = db_session.query(Tag).filter(Tag.name == name).one()
    except NoResultFound:
        abort(404)

    return jsonify(tag.to_dict(max_depth=2))

@bp.route('/tags/<name>', methods=['POST', 'PUT'])
@require_auth_token
def update_tag(auth_token, name):
    try:
        tag = db_session.query(Tag).filter(Tag.name == name).one()
    except NoResultFound:
        abort(404)

    try:
        post_data = request.get_json(force=True)
    except BadRequest as e:
        raise ApiError(e.description)

    tag.update(post_data)

    db_session.commit()

    return jsonify(tag.to_dict(max_depth=2))

@bp.route('/tags/<name>', methods=['DELETE'])
@require_auth_token
def delete_tag(auth_token, name):
    try:
        db_session.query(Tag).filter(Tag.name == name).delete()
    except NoResultFound:
        abort(404)

    db_session.commit()

    return '', 204


import sys
from collections import deque
input = sys.stdin.readline

def init():
    N = int(input().strip())
    e = int(input().strip())
    adj = [[] for _ in range(N+1)]
    for _ in range(e):
        x, y = map(int, input().strip().split())
        adj[x].append(y)
        adj[y].append(x)
    return N, adj

def bfs(adj, start):
    vst = set([start])
    q = deque([start])
    cnt = 0
    while q:
        curr = q.popleft()
        if curr != start:
            cnt += 1
        for nxt in adj[curr]:
            if nxt not in vst:
                q.append(nxt)
                vst.add(nxt)
    return cnt

if __name__ == '__main__':
    start = 1
    N, adj = init()
    print(bfs(adj, start))


import json

import requests

from demo.settings import API_KEY, yunpian_text
# from demo.demo import local_settings
#
# API_KEY = local_settings.API_KEY
# yunpian_text = local_settings.yunpian_text

class YunPian(object):

    def __init__(self):
        self.api_key = API_KEY
        self.single_url = 'https://sms.yunpian.com/v2/sms/single_send.json'

    def send_sms(self, code, mobile):
        """
        发单条短信
        云片文档：https://www.yunpian.com/doc/zh_CN/domestic/single_send.html
        :return: True 发送成功， False 发送失败
        """
        parmas = {
            'apikey': self.api_key,
            'mobile': mobile,
            'text': yunpian_text(code),
        }
        response = requests.post(self.single_url, data=parmas)
        res_dict = json.loads(response.text)
        if res_dict.get(code) != 0:
            # 发送成功
            return True, res_dict.get('msg')
        return False, res_dict.get('msg')

if __name__ == '__main__':
    y = YunPian()
    y.send_sms(2020, 13246855446)


import yaml
import copy
import math
import string

from gestalt.Type import *

class Node(object):
	def __init__(self, classname, name=None, layout=None):
		self.classname = classname
		self.name = None

		self.attrs = {"geometry" : Rect(x=0, y=0, width=0, height=0)}

		if name is not None:
			self.name = name

		if layout is not None:
			self.setProperties(layout)

	def setProperty(self, key, data):
		to_assign = None

		if isinstance(data, DataType):
			to_assign = data
		elif isinstance(data, bool):
			to_assign = Bool(data)
		elif isinstance(data, int):
			to_assign = Number(data)
		elif isinstance(data, float):
			to_assign = Double(data)
		elif isinstance(data, dict):
			to_assign = DataType(data)
		elif isinstance(data, str):
			to_assign = yaml.safe_load(data)

			if not isinstance(to_assign, DataType):
				to_assign = String(data)

		if key in self.attrs:
			self.attrs[key] = self.attrs[key].merge(to_assign)
		else:
			self.attrs[key] = to_assign

		return self

	def setProperties(self, *args, **kwargs):
		if len(args) != 0:
			for key, val in args[0].items():
				self.setProperty(key, val)

		if kwargs:
			for key, val in kwargs.items():
				self.setProperty(key, val)

		return self

	def getProperty(self, key):
		return self.attrs[key]

	def __setitem__(self, key, data):
		self.setProperty(key, data)

	def __getitem__(self, key):
		return self.getProperty(key)

	def position(self, *args, x=None, y=None):
		out_x = 0
		out_y = 0

		if len(args) == 2:
			out_x = args[0]
			out_y = args[1]

		elif x is not None:
			out_x = x
			out_y = y

		elif len(args) == 1:
			if instanceof(args[0], list) or instanceof(args[0], tuple):
				out_x = args[0][0]
				out_y = args[0][1]
			elif instanceof(args[0], dict):
				out_x = args[0]["x"]
				out_y = args[0]["y"]

		self.setProperty("geometry", Rect(x = out_x, y = out_y))

		return self

	def apply (self, generator, data={}):
		return generator.generateWidget(self, macros=data)

class GroupNode(Node):
	def __init__(self, classname, initial=None, name=None, layout=None):
		super(GroupNode, self).__init__(classname, name=name, layout=layout)

		self.children = []

		if initial is not None:
			if isinstance(initial, dict):
				for childname, child in initial.items():
					child.name = childname
					self.append(child)

			elif isinstance(initial, list) or isinstance(initial, tuple):
				for child in initial:
					self.append(child)

	def append(self, child, keep_original=False):
		if not keep_original:
			self.children.append(copy.deepcopy(child))
		else:
			self.children.append(child)

		right_edge  = child["geometry"]["x"] + child["geometry"]["width"]
		bottom_edge = child["geometry"]["y"] + child["geometry"]["height"]

		if right_edge > self["geometry"]["width"]:
			self["geometry"]["width"] = right_edge

		if bottom_edge > self["geometry"]["height"]:
			self["geometry"]["height"] = bottom_edge

		return self

	def apply (self, generator, data={}):
		output = generator.generateGroup(self, macros=data)

		child_macros = copy.deepcopy(data)

		for child in self.children:
			child_macros.update({
				"__parentx__" : output["geometry"]["x"],
				"__parenty__" : output["geometry"]["y"],
				"__parentwidth__" : output["geometry"]["width"],
				"__parentheight__" : output["geometry"]["height"]})

			output.append(child.apply(generator, data=child_macros))

		return output

class GridNode(GroupNode):
	def __init__(self, initial=None, name=None, layout=None, padding=0, repeat=None, ratio=1.0):
		super(GridNode, self).__init__("caFrame", initial=initial, name=name, layout=layout)

		self.ratio = ratio
		self.repeat_over = repeat
		self.padding = padding

	def apply (self, generator, data={}):
		macrolist = data.get(self.repeat_over, {})

		output = generator.generateGroup(self, macros=data)

		num_items = len(macrolist)

		cols = round(math.sqrt(num_items * self.ratio))
		rows = round(math.sqrt(num_items / self.ratio))

		index = 0
		index_x = 0
		index_y = 0

		for macroset in macrolist:
			child_macros = copy.deepcopy(data)
			child_macros.update(macroset)
			child_macros.update({"__index__" : index})

			element = generator.generateAnonymousGroup()

			for childnode in self.children:
				child_macros.update({
					"__parentx__" : output["geometry"]["x"],
					"__parenty__" : output["geometry"]["y"],
					"__parentwidth__" : output["geometry"]["width"],
					"__parentheight__" : output["geometry"]["height"]})

				element.append(childnode.apply(generator, data=child_macros))

			pos_x = index_x * (element["geometry"]["width"] + self.padding)
			pos_y = index_y * (element["geometry"]["height"] + self.padding)

			element.position(pos_x, pos_y)

			index += 1
			index_x += 1

			if index_x >= cols:
				index_x = 0
				index_y += 1

			output.append(element)

		return output

class RepeatNode(GroupNode):
	def __init__(self, initial=None, name=None, layout=None, repeat=None, padding=0, flow="vertical"):
		super(RepeatNode, self).__init__("caFrame", initial=initial, name=name, layout=layout)

		self.repeat_over = repeat
		self.padding = padding
		self.flow = flow

	def apply (self, generator, data={}):
		macrolist = data.get(self.repeat_over, None)

		output = generator.generateGroup(self, macros=data)

		index = 0

		for macroset in macrolist:
			child_macros = copy.deepcopy(data)

			child_macros.update(macroset)

			line = generator.generateAnonymousGroup()

			for childnode in self.children:
				child_macros.update({
					"__parentx__" : self["geometry"]["x"],
					"__parenty__" : self["geometry"]["y"],
					"__parentwidth__" : self["geometry"]["width"],
					"__parentheight__" : self["geometry"]["height"]})

				line.append(childnode.apply(generator, data=child_macros))

			if self.flow == "vertical":
				line.position(x=0, y=(index * (line["geometry"]["height"] + self.padding)))

			elif self.flow == "horizontal":
				line.position(x=(index * (line["geometry"]["width"] + self.padding)), y=0)

			output.append(line)
			index += 1

		return output

class StretchNode(Node):
	def __init__(self, name=None, layout=None, flow="vertical", subnode=None):
		super(StretchNode, self).__init__("Stretch", name=name, layout=layout)

		self.subnode = subnode
		self.flow = flow

	def apply (self, generator, data={}):
		if self.flow == "vertical":
			self.subnode["geometry"]["height"] = data["__parentheight__"]
		elif self.flow == "horizontal":
			self.subnode["geometry"]["width"] = data["__parentwidth__"]

		return self.subnode.apply(generator, data=data)

class CenterNode(Node):
	def __init__(self, name=None, layout=None, flow="vertical", subnode=None):
		super(CenterNode, self).__init__("Center", name=name, layout=layout)

		self.subnode = subnode
		self.flow = flow

	def apply (self, generator, data={}):
		applied_node = self.subnode.apply(generator, data=data)

		if self.flow == "vertical":
			applied_node.position(applied_node["geometry"]["x"], int(data["__parentheight__"] / 2) - int(applied_node["geometry"]["height"] / 2))
		elif self.flow == "horizontal":
			applied_node.position(int(data["__parentwidth__"] / 2) - int(applied_node["geometry"]["width"] / 2), applied_node["geometry"]["y"])

		return applied_node


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.autograd import Variable
import torch.optim as optim
import numpy as np
from pytorch_transformers import *
import time
from torch.utils.data import Dataset, DataLoader
import copy
from random import randrange
from torch.optim import lr_scheduler
import os
import csv
import emoji
import wordsegment
import sys
import nltk
from nltk.corpus import stopwords
from torchtext.data import Field, ReversibleField, Example
from torchtext.data import TabularDataset, Dataset
from torchtext.data import Iterator, BucketIterator
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
from itertools import chain
from torchtext.vocab import GloVe

class Masker():

    def __init__(self, method = 'attention', target = 'OFF', freq_thresh = 5, train_data = 'offenseval-training-v1.tsv', encoder = 'Encoder.pt', classifier = 'Classifier.pt'):

        self.method = method

        if(method == 'attention'):
            self.tokenize = lambda x: nltk.word_tokenize(x.lower())

            self.TEXT = Field(batch_first = True, tokenize = self.tokenize, lower = True, include_lengths=True)
            self.LABEL = Field(sequential = False)
            self.ID = Field(sequential = False, use_vocab = False)

            off_datafields = [('id', None), ('text', self.TEXT), ('label', self.LABEL), ('is_target', None), ('target', None)]

            trn = TabularDataset.splits(path='.', train=train_data, format='tsv', fields=off_datafields)[0]

            self.EMBEDDING_DIM = 200
            self.TEXT.build_vocab(trn, vectors=GloVe(name='6B', dim=self.EMBEDDING_DIM))
            self.LABEL.build_vocab(trn)

            self.BATCH_SIZE = 64

            self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

            if torch.cuda.is_available():
                print ("Using cuda")
                self.classifier = torch.load(classifier)
                self.encoder = torch.load(encoder)
                self.classifier = self.classifier.cuda()
                self.encoder = self.encoder.cuda()
            else:
                print("Using cpu")
                self.classifier = torch.load(classifier, map_location = 'cpu')
                self.encoder = torch.load(encoder, map_location = 'cpu')

        elif(method == 'freq-ratio'):
            # calculates frequencies of tokens for a target attribute
            frequencies = {}

            # smoothing parameter
            lamb = 1

            self.tokenize = lambda x: nltk.word_tokenize(x.lower())
            self.freq_thresh = freq_thresh
            self.target = target

            train_csv = csv.reader(open(train_data, 'r'), delimiter = '\t')

            for example in train_csv:
                text = self.tokenize(example[1])
                for token in text:
                    if token not in frequencies:
                        # stores frequencies of token appearing in target attribute and not in target attribute
                        frequencies[token] = [lamb,lamb]

                    if example[2] == self.target:
                        frequencies[token][0] += 1
                    else:
                        frequencies[token][1] += 1

            self.salience = {}
            for token in frequencies:
                self.salience[token] = frequencies[token][0]/frequencies[token][1]

    # takes in an input sentence and masks words based on the attention,
    # those words higher than the average attention are masked.
    def attention_mask(self, input_text):
        input_fields = [('id', self.ID), ('text', self.TEXT), ('label', self.LABEL)]
        input_example = Example()
        input_example = input_example.fromlist([1, input_text, 1], input_fields)
        input_dataset = Dataset([input_example], input_fields)
        input_iter = BucketIterator(input_dataset, batch_size = 1, device = self.device, repeat = False)

        _, attn = test_model(self.encoder, self.classifier, 1, input_iter)

        attn = attn[0]
        masked_text = self.tokenize(input_text)
        avg = sum(attn)/len(attn)
        for i in range(len(attn)):
            if(attn[i] > avg):
                masked_text[i] = '[MASK]'

        #print(attn)
        return masked_text

    def freq_mask(self, input_text):
        masked_text = self.tokenize(input_text)
        for i in range(len(masked_text)):
            token = masked_text[i]

            # if token has a salience score, check again threshold
            if token in self.salience:
                if self.salience[token] >= self.freq_thresh:
                    masked_text[i] = '[MASK]'

        return masked_text

    def mask(self, input_text):
        if(self.method == 'attention'):
            return self.attention_mask(input_text)
        elif(self.method == 'freq-ratio'):
            return self.freq_mask(input_text)

    # takes in input text and returns corresponding attentions and prediction
    def predict(self, input_text):
        input_fields = [('id', self.ID), ('text', self.TEXT), ('label', self.LABEL)]
        input_example = Example()
        input_example = input_example.fromlist([1, input_text, 1], input_fields)
        input_dataset = Dataset([input_example], input_fields)
        input_iter = BucketIterator(input_dataset, batch_size = 1, device = self.device, repeat = False)

        pred, attn, prob = test_model(self.encoder, self.classifier, 1, input_iter)

        pred = pred[0][1]
        prob = prob[0][1]
        attn = attn[0].tolist()
        attn = [i[0] for i in attn]
        return pred, prob, attn

# Encoder
# Example from https://github.com/nn116003/self-attention-classification/blob/master/imdb_attn.py
class EncoderRNN(nn.Module):

    def __init__(self, emb_dim, h_dim, v_size, gpu=True, v_vec=None, batch_first=True, pad_idx=None):
        super(EncoderRNN, self).__init__()
        self.gpu = torch.cuda.is_available()
        self.h_dim = h_dim
        self.embed = nn.Embedding(v_size, emb_dim, padding_idx = pad_idx)
        if v_vec is not None:
            self.embed.weight.data.copy_(v_vec)
        self.lstm = nn.LSTM(emb_dim, h_dim, batch_first=batch_first,
                            bidirectional=True)

    def init_hidden(self, b_size):
        h0 = Variable(torch.zeros(1*2, b_size, self.h_dim))
        c0 = Variable(torch.zeros(1*2, b_size, self.h_dim))
        if self.gpu and torch.cuda.is_available():
            h0 = h0.cuda()
            c0 = c0.cuda()
        return (h0, c0)

    def forward(self, sentence, lengths = None):
        self.hidden = self.init_hidden(sentence.size(0))
        emb = self.embed(sentence)
        packed_emb = emb

        if lengths is not None:
            lengths = lengths.view(-1).tolist()
            packed_emb = nn.utils.rnn.pack_padded_sequence(emb, lengths)

        out, hidden = self.lstm(packed_emb, self.hidden)

        if lengths is not None:
            out = nn.utils.rnn.pad_packed_sequence(output)[0]

        out = out[:, :, :self.h_dim] + out[:, :, self.h_dim:]

        return out

class Attn(nn.Module):
    def __init__(self, h_dim):
        super(Attn, self).__init__()
        self.h_dim = h_dim
        self.main = nn.Sequential(
            nn.Linear(h_dim, 24),
            nn.ReLU(True),
            nn.Linear(24,1))

    def forward(self, encoder_outputs):
        b_size = encoder_outputs.size(0)
        attn_ene = self.main(encoder_outputs.contiguous().view(-1, self.h_dim)) # (b, s, h) -> (b * s, 1)
        return F.softmax(attn_ene.view(b_size, -1), dim=1).unsqueeze(2) # (b*s, 1) -> (b, s, 1)

class AttnClassifier(nn.Module):
    def __init__(self, h_dim, c_num):
        super(AttnClassifier, self).__init__()
        self.attn = Attn(h_dim)
        self.main = nn.Linear(h_dim, c_num)

    def forward(self, encoder_outputs):
        attns = self.attn(encoder_outputs) #(b, s, 1)
        feats = (encoder_outputs * attns).sum(dim=1) # (b, s, h) -> (b, h)
        return F.log_softmax(self.main(feats)), attns

def binary_accuracy(preds, y):
    """
    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8
    """

    #round predictions to the closest integer
    rounded_preds = torch.round(torch.sigmoid(preds))
    correct = (rounded_preds == y).float() #convert into float for division
    acc = correct.sum() / len(correct)
    return acc

def train_model(encoder, classifier, epoch, train_iter, optimizer, batch_size, log_interval=10):
    encoder.train()
    classifier.train()
    correct = 0

    for idx, batch in enumerate(train_iter):
        (x, x_l), y = batch.text, batch.label - 1
        #print(y)
        optimizer.zero_grad()
        encoder_outputs = encoder(x)
        output, attn = classifier(encoder_outputs)
        loss = F.nll_loss(output, y)
        loss.backward()
        optimizer.step()

        #print(output)

        pred = output.data.max(1, keepdim=True)[1]
        #print(pred)
        correct += int(pred.eq(y.data.view_as(pred)).cpu().sum())
        #print(correct)
        if idx % log_interval == 0:
            print('train epoch: {} [{}/{}], acc:{}, loss:{}'.format(
                epoch, idx*len(x), len(train_iter)*batch_size,
                correct/float(log_interval * len(x)),
                loss.data))
            correct = 0

def test_model(encoder, classifier, epoch, test_iter):
    encoder.eval()
    classifier.eval()
    correct = 0
    total = 0
    final_preds = []
    pred_probs = []
    attentions = []
    softmax = torch.nn.Softmax()

    for idx, batch in enumerate(test_iter):
        (x, x_l), y = batch.text, batch.label - 1

        encoder_outputs = encoder(x)
        output, attn = classifier(encoder_outputs)
        pred = output.data.max(1, keepdim=True)[1]

        for i in range(len(batch.id)):
            soft_probs = softmax(output[i])
            final_preds.append((batch.id[i].item(), pred[i].item()))
            pred_probs.append((batch.id[i].item(), soft_probs[1].item()))
            #print(attn[i])
            attentions.append(attn[i])

    return final_preds, attentions, pred_probs

        #correct += int(pred.eq(y.data.view_as(pred)).cpu().sum())
        #print(correct)
        #total += int(len(batch))
        #print(total)

    #print('test epoch:{}, acc:{}'.format(epoch, correct/float(total)))

def epoch_time(start_time, end_time):
    elapsed_time = end_time - start_time
    elapsed_mins = int(elapsed_time / 60)
    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
    return elapsed_mins, elapsed_secs

def main(trainFile, testFile, train_test, out_file = None):

    tokenize = lambda x: nltk.word_tokenize(x.lower())

    TEXT = Field(batch_first = True, tokenize = tokenize, lower = True, include_lengths=True)
    LABEL = Field(sequential = False)
    ID = Field(sequential = False, use_vocab = False)

    off_datafields = [('id', None), ('text', TEXT), ('label', LABEL), ('is_target', None), ('target', None)]

    trn = TabularDataset.splits(path='.', train=trainFile, format='tsv', fields=off_datafields)[0]

    #for item in trn:
    #    print(trn)

    tst_datafields = [('id', ID), ('text', TEXT), ('label', LABEL)]

    tst = TabularDataset(path=testFile, format='tsv', fields = tst_datafields)

    for item in tst:
        print(item)

    EMBEDDING_DIM = 200

    TEXT.build_vocab(trn, vectors=GloVe(name='6B', dim=EMBEDDING_DIM))
    LABEL.build_vocab(trn)

    BATCH_SIZE = 64

    device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

    train_iterator = BucketIterator(trn, batch_size = BATCH_SIZE, device = device, repeat = False)#sort_key = lambda x: len(x.text), sort_within_batch = True)

    test_iterator = BucketIterator(tst, batch_size=BATCH_SIZE, device = device, repeat = False)#sort_key = lambda x: len(x.text), sort_within_batch=True)

    if(train_test == 'train' or train_test == 'both'):

        # train BLSTM
        #

        INPUT_DIM = len(TEXT.vocab)
        HIDDEN = 64
        OUTPUT_DIM = 2
        DROPOUT = 0.2
        PAD_IDX = TEXT.vocab.stoi[TEXT.pad_token]

        encoder = EncoderRNN(EMBEDDING_DIM, HIDDEN, INPUT_DIM, gpu=torch.cuda.is_available(), v_vec = TEXT.vocab.vectors, pad_idx = PAD_IDX)

        classifier = AttnClassifier(HIDDEN, OUTPUT_DIM)

        UNK_IDX = TEXT.vocab.stoi[TEXT.unk_token]

        encoder.embed.weight.data[UNK_IDX] = torch.zeros(EMBEDDING_DIM)
        encoder.embed.weight.data[PAD_IDX] = torch.zeros(EMBEDDING_DIM)

        # init model
        def weights_init(m):
            classname = m.__class__.__name__
            if hasattr(m, 'weight') and (classname.find('Embedding') == -1):
                nn.init.xavier_uniform(m.weight.data, gain=nn.init.calculate_gain('relu'))

        for m in encoder.modules():
            print(m.__class__.__name__)
            weights_init(m)

        for m in classifier.modules():
            print(m.__class__.__name__)
            weights_init(m)

        optimizer = optim.Adam(chain(encoder.parameters(),classifier.parameters()), lr=0.001)

        encoder =  encoder.to(device)
        classifier = classifier.to(device)

        N_EPOCHS = 5

        best_valid_loss = float('inf')

        print('training BLSTM')
        for epoch in range(N_EPOCHS):

            start_time = time.time()

            #train_loss, train_acc =
            train_model(encoder, classifier, epoch + 1, train_iterator, optimizer, BATCH_SIZE)
            #valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)

            #test_model(encoder, classifier, epoch + 1, test_iterator)

            end_time = time.time()

            epoch_mins, epoch_secs = epoch_time(start_time, end_time)

            #if valid_loss < best_valid_loss:
            #    best_valid_loss = valid_loss
            #    torch.save(model.state_dict(), 'tut4-model.pt')

            print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
            #print(f'\tTrain Loss: {train_loss:.3f}')
            #print(f'\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')

        torch.save(encoder, 'Encoder.pt')
        torch.save(classifier, 'Classifier.pt')

    elif(train_test == 'test' or train_test == 'both'):

        #BLSTM
        encoder = torch.load('Encoder.pt')
        classifier = torch.load('Classifier.pt')

        preds, _ = test_model(encoder, classifier, 1, test_iterator)

        pred_dict = {}
        for id, pred in preds:
            pred_dict[str(id)] = pred

        if(out_file):
            output = open(out_file, 'w')
        else:
            output = open('BLSTM_Attention_predictionsOut', 'w')

        for id in pred_dict:
            pred = pred_dict[id]
            output.write(str(id) + ',' + str(pred) + '\n')

        output.close()

if(__name__ == "__main__"):
    if(len(sys.argv) == 4):
        main(sys.argv[1], sys.argv[2], sys.argv[3])
    else:
        main(sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4])


import streamlit as st
import pandas as pd

FILE_PATH = "salesweekly.csv"

def read_file(FILEPATH):
    return pd.read_csv(FILEPATH)

def main():
    st.title("Here is our homepage")
    df = read_file(FILE_PATH)
    if st.checkbox("show data"):
        st.table(df.head())

if __name__ == "__main__":
    main()


import sys
from collections import deque
sys.stdin = open("./baekjoon/testcase.txt")

N, K = tuple(map(int, sys.stdin.readline().split()))

def BFS(s):
    parent = {s} # set
    frontier = deque([(s, 0)])
    while frontier:
        v1, time1 = frontier.popleft()
        # 한 노드에서 이동할 수 있는 경우 3가지
        next_nodes = [v1+1,v1-1]
        if v1 % 2 == 0:
            next_nodes.append(v1//2)
        for v2 in next_nodes:
            time2 = time1+1
            # 이동 불가능한 지점인지 확인
            if v2 <0:
                continue
            # 이미 방문한 위치인지 확인
            if v2 in parent:
                continue
            # 새로운 위치인 경우, 우선 목표 지점인지 확인
            if v2 == N:
                return time2
            # 새로운 위치이지만 목표 지점이 아닌 경우, 탐색 범위에 추가
            parent.add(v2)
            frontier.append((v2, time2))
    return False

s = K
if s == N:
    print(0)
else:
    time = BFS(s)
    print(time)


import json

from flask.ext.jsontools import JsonSerializableBase
from sqlalchemy import BigInteger, Column, Integer, TIMESTAMP, String, Text, Boolean
from sqlalchemy.dialects.postgresql import JSONB
from dart.model.action import Action
from dart.model.accounting import Accounting
from dart.model.authorization_roles import AuthorizationRoles
from dart.model.user_roles import UserRoles

from dart.model.dataset import Dataset
from dart.model.datastore import Datastore
from dart.context.database import db
from dart.model.engine import Engine
from dart.model.event import Event
from dart.model.graph import SubGraphDefinition
from dart.model.message import Message
from dart.model.mutex import Mutex
from dart.model.subscription import Subscription, SubscriptionElement
from dart.model.trigger import Trigger
from dart.model.user import User
from dart.model.api_key import ApiKey
from dart.model.workflow import Workflow, WorkflowInstance
from dart.util.json_util import DartJsonEncoder

class VersionedAuditableSerializable(JsonSerializableBase):
    id = Column(String(length=36), primary_key=True)
    version_id = Column(Integer, index=True, nullable=False, server_default='0')
    created = Column(TIMESTAMP, server_default=db.func.current_timestamp())
    updated = Column(TIMESTAMP, server_default=db.func.current_timestamp(), onupdate=db.func.current_timestamp())
    __mapper_args__ = {"version_id_col": version_id}
    __modelclass__ = None

    def to_model(self):
        return self.__modelclass__.from_dict(json.loads(json.dumps(self, cls=DartJsonEncoder)))

class VersionedAuditableData(VersionedAuditableSerializable):
    data = Column(JSONB)

class AuthorizationRolesDao(db.Model):
    __tablename__ = 'authorization_roles'
    __modelclass__ = AuthorizationRoles
    role_id = Column(String(length=36), primary_key=True)
    role_name = Column(String(length=255), unique=True, nullable=False)

class UserRolesDao(db.Model):
    __tablename__ = 'user_roles'
    __modelclass__ = UserRoles
    user_id = Column(String(length=36), primary_key=True)
    action_roles = Column(JSONB)
    membership_roles = Column(JSONB)

class EngineDao(db.Model, VersionedAuditableData):
    __tablename__ = 'engine'
    __modelclass__ = Engine
    name = Column(String(length=255), unique=True, nullable=False)

class SubGraphDefinitionDao(db.Model, VersionedAuditableData):
    __tablename__ = 'subgraph_definition'
    __modelclass__ = SubGraphDefinition

class DatasetDao(db.Model, VersionedAuditableData):
    __tablename__ = 'dataset'
    __modelclass__ = Dataset
    name = Column(String(length=255), unique=True, nullable=False)

class ActionDao(db.Model, VersionedAuditableData):
    __tablename__ = 'action'
    __modelclass__ = Action

class DatastoreDao(db.Model, VersionedAuditableData):
    __tablename__ = 'datastore'
    __modelclass__ = Datastore

class TriggerDao(db.Model, VersionedAuditableData):
    __tablename__ = 'trigger'
    __modelclass__ = Trigger

class WorkflowDao(db.Model, VersionedAuditableData):
    __tablename__ = 'workflow'
    __modelclass__ = Workflow

class WorkflowInstanceDao(db.Model, VersionedAuditableData):
    __tablename__ = 'workflow_instance'
    __modelclass__ = WorkflowInstance

class EventDao(db.Model, VersionedAuditableData):
    __tablename__ = 'event'
    __modelclass__ = Event

class SubscriptionDao(db.Model, VersionedAuditableData):
    __tablename__ = 'subscription'
    __modelclass__ = Subscription

class SubscriptionElementDao(db.Model, VersionedAuditableSerializable):
    __tablename__ = 'subscription_element'
    __modelclass__ = SubscriptionElement
    subscription_id = Column(String(length=36), nullable=False)
    s3_path = Column(String(length=1024), index=True, nullable=False)
    file_size = Column(BigInteger, nullable=False)
    state = Column(String(length=50), nullable=False)
    action_id = Column(String(length=36))
    batch_id = Column(String(length=36))
    processed = Column(TIMESTAMP)

class MessageDao(db.Model, VersionedAuditableSerializable):
    __tablename__ = 'message'
    __modelclass__ = Message
    message_body = Column(Text(), nullable=False)
    instance_id = Column(String(length=32), nullable=False)
    container_id = Column(String(length=256), nullable=False)
    ecs_cluster = Column(String(length=70), nullable=False)
    ecs_container_instance_arn = Column(String(length=256), nullable=False)
    batch_job_id = Column(String(length=256), nullable=False)
    ecs_task_arn = Column(String(length=256), nullable=False)
    state = Column(String(length=50), index=True, nullable=False)

class AccountingDao(db.Model, VersionedAuditableSerializable):
    __tablename__ = 'accounting'
    __modelclass__ = Accounting
    user_id = Column(String(length=128), nullable=False)
    state = Column(String(length=32), nullable=False)
    entity = Column(String(length=32), nullable=False)
    params = Column(String(length=1024), nullable=False)
    return_code = Column(String(length=4), nullable=False)
    api_version = Column(String(length=4), nullable=False)
    extra = Column(String(length=128), nullable=True)

class MutexDao(db.Model, VersionedAuditableSerializable):
    __tablename__ = 'mutex'
    __modelclass__ = Mutex
    name = Column(String(length=255), unique=True, nullable=False)
    state = Column(String(length=50), nullable=False)

class UserDao(db.Model, VersionedAuditableSerializable):
    __tablename__ = 'user'
    __modelclass__ = User
    email = Column(String(length=255), unique=True, nullable=False)
    first_name = Column(String(length=255), unique=False, nullable=True)
    last_name = Column(String(length=255), unique=False, nullable=True)
    is_authenticated = Column(Boolean, server_default='false', nullable=False)
    session_expiration = Column(TIMESTAMP, server_default=db.func.current_timestamp())

class ApiKeyDao(db.Model, VersionedAuditableSerializable):
    __tablename__ = 'api_key'
    __modelclass__ = ApiKey
    user_id = Column(String(length=36), unique=False, nullable=False)
    api_key = Column(String(length=255), unique=True, nullable=False)
    api_secret = Column(String(length=255), unique=False, nullable=False)


import pytest
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import load_iris

from dabl.utils import data_df_from_bunch
from dabl.plot.utils import find_pretty_grid, plot_coefficients, pairplot

def test_find_pretty_grid():
    # test that the grid is big enough:
    rng = np.random.RandomState(0)
    for i in range(100):
        n_plots = rng.randint(1, 34)
        max_cols = rng.randint(1, 12)
        rows, cols = find_pretty_grid(n_plots=n_plots, max_cols=max_cols)
        assert rows * cols >= n_plots
        assert cols <= max_cols

@pytest.mark.parametrize("n_features, n_top_features",
                         [(5, 10), (10, 5), (10, 40), (40, 10)])
def test_plot_coefficients(n_features, n_top_features):
    coef = np.arange(n_features) + .4
    names = ["feature_{}".format(i) for i in range(n_features)]
    plot_coefficients(coef, names, n_top_features=n_top_features)
    ax = plt.gca()
    assert len(ax.get_xticks()) == min(n_top_features, n_features)
    coef[:-5] = 0
    plot_coefficients(coef, names, n_top_features=n_top_features)
    ax = plt.gca()
    assert len(ax.get_xticks()) == 5

def test_pairplot_iris():
    data = data_df_from_bunch(load_iris())
    axes = pairplot(data, target_col='target')
    # the diagonal axes are duplicated
    # so we can have sharex / sharey
    assert axes.shape == (4, 4)
    assert len(plt.gcf().get_axes()) == 4 * 4 + 4


from datetime import timedelta
from time import sleep

from django.db import transaction
from django.db.transaction import rollback
from django.http import JsonResponse, HttpResponse
from django.template.backends import django
from django.utils.datetime_safe import datetime
from rest_framework import serializers

from wwwApp.models import Flight, Crew
from wwwApp.utils import flight_intersect_crew_flights, intersect

class Request:
    pass

@transaction.atomic
def SynchronizeWebService(request):
    try:
        print("Poczatek synchronize")
        sid = transaction.savepoint()
        requests = []
        for i,(key,value) in enumerate(request.GET.items()):
            if (i % 2 == 1):
                requests[-1].flight_id = int(value)
            else:
                requests.append(Request())
                if (value == 'remove'):
                    requests[-1].remove = True
                else:
                    requests[-1].remove = False

                    requests[-1].crew_id = int(value)

        sid = transaction.savepoint()
        wrong_flights = set()
        for r in requests:
            flight = Flight.objects.get(id=r.flight_id)
            if (r.remove):
                flight.crew = None
                flight.save()
            else:
                crew = Crew.objects.get(id=r.crew_id)
                flight.crew = crew
                flight.save()

        for r in requests:
            if (r.remove == False):
                crew = Crew.objects.get(id=r.crew_id)
                crew_flights = Flight.objects.filter(crew=crew)
                for flight1 in crew_flights:
                    for flight2 in crew_flights:
                        if (flight1 != flight2 and intersect(flight1,flight2)):
                            wrong_flights.add(flight1)
                            wrong_flights.add(flight2)
        sleep(3)
        if (wrong_flights):
            transaction.savepoint_rollback(sid)

        serializer = FlightsSerializer(wrong_flights, many=True)
        print("Koniec synchronize2")
        return JsonResponse(serializer.data, safe=False)
    except:
        print("Ktos w tym czasie synchronizuje")
        return JsonResponse(dict(busy='busy'), safe=False)

class CrewSerializer(serializers.ModelSerializer):
    class Meta:
        model = Crew
        fields = '__all__'

def CrewRestWebService(request):
    crews = Crew.objects.all()
    serializer = CrewSerializer(crews, many=True)
    return JsonResponse(serializer.data, safe=False)

class FlightsSerializer(serializers.ModelSerializer):
    pass

    class Meta:
        model = Flight
        fields = (
            'id', 'starting_airport_name', 'starting_time', 'destination_airport_name', 'destination_time', 'crew_name',
            'starting_time_formatted', 'destination_time_formatted')

def FlightRestWebService(request):
    if 'date' in request.GET:
        flights = flights_filtered_by_date(request)
    else:
        flights = Flight.objects.all()
    serializer = FlightsSerializer(flights, many=True)
    return JsonResponse(serializer.data, safe=False)

def flights_filtered_by_date(request):
    format = '%Y-%m-%d'
    date_str = request.GET["date"]
    day = datetime.strptime(date_str, format)
    next_day = day + timedelta(days=1)
    flights = Flight.objects.filter(starting_time__range=[day.strftime(format), next_day.strftime(format)])
    return flights


from django.urls import path
from . import views
from django.conf import settings
from django.conf.urls.static import static

urlpatterns = [
    path('', views.home, name='home'),
    path('login/', views.login_user, name='login'),
    path('logout/', views.logout_user, name='logout'),
    path('tasks/', views.tasks, name='tasks'),
    path('add_task/', views.add_task, name='add_task'),
    path('load_topics/', views.load_topics, name='load_topics'),
    path('load_types/', views.load_types, name='load_types'),
    path('tasks/<task_id>', views.task_page, name='task_page_by_id'),
    path('assignments/', views.assignments, name='assignments'),
    path('assignments/<assignment_id>', views.assignment_page, name='assignment_page_by_id'),
    path('add_task_to_cart/', views.add_task_to_cart, name='add_task_to_cart'),
    path('remove_task_from_cart/', views.remove_task_from_cart, name='remove_task_from_cart'),
    path('cart/', views.cart, name='cart'),
    path('search_user/', views.search_user, name='search_user'),
    path('upload_response/<assignment_id>', views.upload_response, name='upload_response'),
    path('assigned_by_me/', views.assigned_by_me, name='assigned_by_me'),
    path('lessons/', views.lessons, name="lessons"),
    path('lessons/add', views.add_lesson, name="add_lesson"),
    path('upload_temp_file/', views.upload_temp_file, name="upload_temp_file"),
    path('generatePDF_from_tasks/', views.generate_pdf_from_tasks, name="generatePDF_from_tasks"),
    path('download_pdf/', views.download_pdf, name='download_pdf'),
    path('cart/clear/', views.clear_cart, name="clear_cart"),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)


# Databricks notebook source
# MAGIC %md
# MAGIC # Stateful aggregation alternatives
# MAGIC Scenario: Suppose you would do a running count using stateful streaming aggregations, would there be an alternative option to do this without relying on the state store ?

# COMMAND ----------

# MAGIC %run "./includes/setup"

# COMMAND ----------

import pyspark.sql.types as T
from delta.tables import DeltaTable
from pyspark.sql import DataFrame
import pyspark.sql.functions as F

# Prevent the creation of unnecessary small files.
spark.conf.set("spark.databricks.delta.properties.defaults.autoOptimize.optimizeWrite", True)
spark.conf.set("spark.databricks.delta.properties.defaults.autoOptimize.autoCompact", True)

# Use RocksDB as state store backend.
spark.conf.set("spark.sql.streaming.stateStore.providerClass", "com.databricks.sql.streaming.state.RocksDBStateStoreProvider")

# Clean out old runs.
clean_workspace(user_home)

# COMMAND ----------

# MAGIC %md-sandbox ## Stateful Operator Implementation
# MAGIC
# MAGIC This is roughly what is happening when trying to compute a **groupBy+count** using the stateful API. It will store intermediate counts in the state store and keep joining this to the new data coming in. The number of partitions used in performing the logical join between the new data and existing state will be fixed after stream start (tied to checkpoint), and is tunable using **spark.sql.shuffle.partitions** (default 200).
# MAGIC
# MAGIC

# COMMAND ----------

stateful_output_path = "{}/stateful_aggregation_alternatives/stateful_operator".format(user_home)
stateful_output_checkpoint = "{}/_checkpoint".format(stateful_output_path)

until_stream_is_ready(
    spark
    .readStream
    .format("rate")
    .option("rowsPerSecond", 50)
    .load()
    .withColumn("key", F.col("value") % 50) # Keep generating repeating keys.
    .groupBy("key")
    .count()                                  # This is stateful aggregation relying on the state store. If checkpoint needs to be reset (for whatever reason) -> state gone, needs to be rebuild.
    .writeStream
    .format("delta")
    .outputMode("complete")
    .option("checkpointLocation", stateful_output_checkpoint) # State is stored here.
    .start(stateful_output_path)

)

# COMMAND ----------

# MAGIC %md ### Output

# COMMAND ----------

display(DeltaTable.forPath(spark, stateful_output_path).toDF())

# COMMAND ----------

# MAGIC %md ### Output directory (Delta Lake Table)

# COMMAND ----------

display(dbutils.fs.ls(stateful_output_path))

# COMMAND ----------

# MAGIC %md ### Checkpoint directory

# COMMAND ----------

display(dbutils.fs.ls(stateful_output_checkpoint))

# COMMAND ----------

# MAGIC %md ### State Store files for each of the 200 partitions (output shows partition 0)

# COMMAND ----------

display(dbutils.fs.ls("{}/state/0/0".format(stateful_output_checkpoint)))

# COMMAND ----------

t = DeltaTable.forPath(spark, stateful_output_path)
display(t.history())

# COMMAND ----------

# MAGIC %md-sandbox ## Alternative Implementation (using MERGE instead of Stateful Operators)
# MAGIC
# MAGIC This implementation does not rely on the state store. Instead it MERGEs the input with the already existing output to update the counts.
# MAGIC
# MAGIC

# COMMAND ----------

# MAGIC %md ### Create Delta table that stores state (instead of the state store)

# COMMAND ----------

alternative_output_path = "{}/stateful_aggregation_alternatives/alternative".format(user_home)
alternative_output_checkpoint = "{}/_checkpoint".format(alternative_output_path)

alternative_delta_schema = (
  T.StructType([
    T.StructField("key", T.LongType()),
    T.StructField("count", T.LongType())
])
)

# Create an empty Delta table if it does not exist. This is required for the MERGE to work in the first mini batch.
if not DeltaTable.isDeltaTable(spark, alternative_output_path):
  (
    spark
      .createDataFrame([], alternative_delta_schema)
      .write
      .mode("overwrite")
      .option("overwriteSchema", "true")
      .format("delta")
      .save(alternative_output_path)
  )

# COMMAND ----------

# MAGIC %md ### Define the alternative to groupBy+count

# COMMAND ----------

def group_by_count(updates_df: DataFrame, epoch_id: int):

  # Count the new incoming keys.
  incoming_counts_df = updates_df.groupBy("key").count()

  target_table = DeltaTable.forPath(spark, alternative_output_path)

  # We merge the new counts with the already existing counts.
  (
    target_table.alias("target").merge(
      incoming_counts_df.alias("source"),
      "target.key = source.key")
    .whenMatchedUpdate(set = { "count" : F.col("source.count") + F.col("target.count") } )
    .whenNotMatchedInsertAll()
    .execute()
  )

# COMMAND ----------

# MAGIC %md ### Run the alternative

# COMMAND ----------

until_stream_is_ready(
    spark
    .readStream
    .format("rate")
    .option("rowsPerSecond", 50)
    .load()
    .withColumn("key", F.col("value") % 50) # Keep generating repeating keys.
    .writeStream
    .foreachBatch(group_by_count)
    .outputMode("append")
    .option("checkpointLocation", alternative_output_checkpoint) # Only commits/offsets are stored here, not the actual state of the counts.
    .start(alternative_output_path)
)

# COMMAND ----------

# MAGIC %md ### Alternative Output

# COMMAND ----------

display(DeltaTable.forPath(spark, alternative_output_path).toDF().orderBy("count", ascending=False))

# COMMAND ----------

# MAGIC %md ### Alternative Output directory (Delta Lake Table)

# COMMAND ----------

display(dbutils.fs.ls(alternative_output_path))

# COMMAND ----------

# MAGIC %md ### Alternative Checkpoint directory (see that "state" sub directory is not present)

# COMMAND ----------

display(dbutils.fs.ls(alternative_output_checkpoint))

# COMMAND ----------

# MAGIC %md # Clean up

# COMMAND ----------

for s in spark.streams.active: # Iterate over all active streams
  s.stop()                     # Stop the stream

# Clean out old runs.
clean_workspace(user_home)

# COMMAND ----------

# MAGIC %md # Summary of benefits
# MAGIC
# MAGIC ## Stateful Operator
# MAGIC 1. In-memory backed state store (RocksDB) which can be faster than going to disk.
# MAGIC 2. Code is more concise.
# MAGIC 3. Natively supports dropping of late data.
# MAGIC
# MAGIC ## Alternative
# MAGIC 1. State is queryable as a Delta table.
# MAGIC 2. State will not need to be rebuilt after [making changes that require a wipe of the checkpoint](https://docs.databricks.com/spark/latest/structured-streaming/production.html#recover-after-changes-in-a-streaming-query).
# MAGIC 3. State can grow a lot without requiring a super big cluster (make sure it is manageable though, or you might end up rewriting a lot of data after every mini batch).
# MAGIC 4. Shuffle partitions used in MERGE can be adapted without requiring a checkpoint restrat (compared to them being fixed in the stateful operator implementation).
# MAGIC 4. Can be cheaper as state store backing stateful operator generates a lot of small files, which can incur higher object store / list / put cost.


import math
import os.path
import random
import sys
import pygame

pygame.init()

WIDTH = 800
HEIGHT = 500

pygame.display.set_caption("Hangman Game")
project_directory = os.path.dirname(__file__)

"""Author https://www.iconbros.com/icons/ib-g-hangman for icon"""
ICON = pygame.image.load(os.path.join(project_directory, "images/icon.png"))
pygame.display.set_icon(ICON)

# Font
COMICSHANNS = os.path.join(project_directory, "font/comic-shanns.otf")

TITLE_FONT = pygame.font.Font(COMICSHANNS, 70)
LETTER_FONT = pygame.font.Font(COMICSHANNS, 30)
WORD_FONT = pygame.font.Font(COMICSHANNS, 40)

# Game variables
HANGMAN_STATUS = 0
WORDS = [
    "IDE",
    "PYTHON",
    "DEVELOPER",
    "PYGAME",
    "VIM",
    "STRAGER",
    "RUBY",
    "CPLUSPLUS",
    "TWITCH",
    "JAVASCRIPT",
    "HTML",
    "CSS",
    "NODEJS",
]
WORD = random.choice(WORDS)
GUESSED = ["D", "Y", "I", "J", "A", "H"]
CLICK = False

# Buttons
RADIUS = 20
GAP = 15
letters = []
STARTX = round((WIDTH - (RADIUS * 2 + GAP) * 13) / 2)  # answer = 42
STARTY = 400
A = 65
for i in range(26):
    x = STARTX + GAP * 2 + ((RADIUS * 2 + GAP) * (i % 13))
    y = STARTY + ((i // 13) * (GAP + RADIUS * 2))
    letters.append([x, y, chr(A + i), True])

# Colors
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
BRIGHT_GREY = (60, 60, 90)
GREEN = (66, 244, 66)
RED = (255, 0, 0)

def draw():
    screen.fill(WHITE)

    # Text
    text = TITLE_FONT.render("DEVELOPER HANGMAN", True, BLACK)
    screen.blit(text, (int(WIDTH / 2) - int(text.get_width() / 2), 20))

    # Hangman image
    screen.blit(images[HANGMAN_STATUS], (150, 100))

    # Draw buttons
    for letter in letters:
        a, b, ltr, visible = letter
        if visible:
            pygame.draw.circle(screen, BLACK, (a, b), RADIUS, 3)
            text = LETTER_FONT.render(ltr, True, BLACK)
            screen.blit(
                text,
                (int(a - text.get_width() / 2), int(b - text.get_height() / 2)),
            )

    # Draw word
    display_word = ""
    for letter in WORD:
        if letter in GUESSED:
            display_word += letter + " "
        else:
            display_word += "_ "
    text = WORD_FONT.render(display_word, True, BLACK)
    screen.blit(text, (400, 200))

def display_message(message):
    global WORD

    screen.fill(WHITE)

    # You won or You lose
    text = WORD_FONT.render(message, True, BLACK)
    screen.blit(
        text,
        (
            int(WIDTH / 2 - text.get_width() / 2),
            int(HEIGHT / 2 - text.get_height() / 2),
        ),
    )

    # The was word
    display_word = WORD_FONT.render(f"The word was {WORD}", True, BLACK)
    screen.blit(
        display_word,
        (
            int(WIDTH / 2 - display_word.get_width() / 2),
            int(HEIGHT / 2 - display_word.get_height() / 2 + 50),
        ),
    )
    pygame.display.update()

def menu():
    global CLICK, HANGMAN_STATUS, GUESSED, WORDS, WORD

    # FPS
    fps = 60
    clock = pygame.time.Clock()

    while True:
        clock.tick(fps)
        pos_x, pos_y = pygame.mouse.get_pos()

        # Position of buttons
        pos_play_button = (100, 400, 120, 50)
        pos_quit_button = (550, 400, 120, 50)

        # Create the buttons
        button_play = pygame.Rect(pos_play_button)
        button_quit = pygame.Rect(pos_quit_button)

        # Create text inside the buttons
        text_play = WORD_FONT.render("Play", True, BLACK)
        text_quit = WORD_FONT.render("Quit", True, BLACK)

        # Check collision
        if button_play.collidepoint(pos_x, pos_y):
            pygame.draw.rect(screen, BRIGHT_GREY, pos_play_button)
            if CLICK:
                HANGMAN_STATUS = 0
                WORDS = ["IDE", "PYTHON", "DEVELOPER", "PYGAME", "VIM"]
                WORD = random.choice(WORDS)
                GUESSED = ["D", "Y", "I"]
                CLICK = False
                for letter in letters:
                    letter[3] = True
                main()
        if button_quit.collidepoint(pos_x, pos_y):
            pygame.draw.rect(screen, BRIGHT_GREY, pos_quit_button)
            if CLICK:
                sys.exit()

        # Draw the buttons
        pygame.draw.rect(screen, GREEN, button_play)
        pygame.draw.rect(screen, RED, button_quit)

        screen.blit(text_play, (120, 405))
        screen.blit(text_quit, (570, 405))

        CLICK = False

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            if event.type == pygame.MOUSEBUTTONDOWN:
                if event.button == 1:
                    CLICK = True

        pygame.display.update()

def main():
    global HANGMAN_STATUS

    # FPS
    fps = 60
    clock = pygame.time.Clock()
    run = True

    while run:
        # FPS
        clock.tick(fps)

        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                sys.exit()
            if event.type == pygame.MOUSEBUTTONDOWN:
                m_x, m_y = pygame.mouse.get_pos()
                for letter in letters:
                    j, k, ltr, visible = letter
                    if visible:
                        dis = math.sqrt((j - m_x) ** 2 + (k - m_y) ** 2)
                        if dis < RADIUS:
                            letter[3] = False
                            GUESSED.append(ltr)
                            if ltr not in WORD:
                                HANGMAN_STATUS += 1
        draw()

        won = True
        for letter in WORD:
            if letter not in GUESSED:
                won = False
                break

        if won:
            display_message("You won")
            menu()
            break

        if HANGMAN_STATUS == 6:
            display_message("You lose")
            menu()
            break

        pygame.display.update()

if __name__ == "__main__":
    screen = pygame.display.set_mode((WIDTH, HEIGHT))

    # Images
    images = []
    for i in range(7):
        image = os.path.join(project_directory, "images/hangman")
        image = pygame.image.load(image + str(i) + ".png")
        images.append(image)

    main()


import numpy as np
import os
import time
import scipy.stats as ss
import os
import argparse
import sys
cdir = os.path.dirname(os.path.dirname(sys.path[0]))

np.random.seed(0)
start = time.time()
# parameters for our spin (effective spin,  effective precessing spin) population, which is a multivariate normal distribution + peak
spin_pars = {'mu_eff': 0.06, # mean of effective spin
  'sigma_eff': 0.12, # standard deviation of effective spin
  'mu_p': 0.21, # mean of effective precessing spin
  'sigma_p': 0.09, # standard deviation of effective precessing spin
  'rho': 0.12, # parameter in for the covariance matrix
  'lambda':0.2, # fraction of the peak
  'peak1_mean':0.3,
  'peak2_mean':0.35,
  'peak_sd':0.03}
def TruncNormSampler(clip_a, clip_b, mean, std, Nsamples):
    a, b = (clip_a - mean) / std, (clip_b - mean) / std
    return ss.truncnorm.rvs(a,b,size=Nsamples ) * std + mean

class Gaussian_spin_distribution():
    def __init__(self, peak = False, **model_pars):
        mu_eff = model_pars['mu_eff']
        mu_p = model_pars['mu_p']
        sigma_eff = model_pars['sigma_eff']
        sigma_p = model_pars['sigma_p']
        rho = model_pars['rho']
        self.peak = peak
        self.lambda_peak= model_pars['lambda']
        self.peak1_mean = model_pars['peak1_mean']
        self.peak2_mean = model_pars['peak2_mean']
        self.peak_sd = model_pars['peak_sd']
        self.eff_bounds=[-1,1]
        self.p_bounds=[0,1]

        self.mean = np.array([mu_eff, mu_p])
        self.cov = np.array([[sigma_eff**2, rho*sigma_eff*sigma_p],[rho*sigma_eff*sigma_p,sigma_p**2]])

        self.model=ss.multivariate_normal(mean=self.mean, cov=self.cov)
        self.normalization = [1,1]
#        self.normalization = self.GetNormalization()
#        print(self.normalization)

    def GetGrid(self, n_pts=2000):
        xeff  = np.linspace(self.eff_bounds[0], self.eff_bounds[1], n_pts)
        xp = np.linspace(self.p_bounds[0],self.p_bounds[1],n_pts)
        xeff_grid, xp_grid = np.meshgrid(xeff, xp,indexing='ij')
        return xeff_grid.flatten(), xp_grid.flatten(), xeff, xp
    """
    def GetNormalization(self):
        x,y = self.GetGrid()[:2]
        if self.peak:
            return [2*np.mean(self.model.pdf(np.array([x,y]).T)),2*np.mean(self.narrow_peak_pdf(np.array([x,y])))]
        else:
            return [2*np.mean(self.model.pdf(np.array([x,y]).T))]
    """
    def sample(self, Nsamples):

        if self.peak:
            Npeak = int(Nsamples * self.lambda_peak)
            Nnormal = Nsamples - Npeak
            chi_eff, chi_p = self.model.rvs(size=Nnormal).T
            while True:
                index = np.where((chi_eff>1) + (chi_eff<-1) + (chi_p>1) + (chi_p<0)   )[0]
                n_out = index.size
                if not n_out>0: break
                chi_eff[index], chi_p[index] = self.model.rvs(size=n_out).T
            chi_eff = np.append(chi_eff, self.narrow_peak_rvs(self.peak1_mean, self.peak_sd, size=Npeak) )
            chi_p = np.append(chi_p, self.narrow_peak_rvs(self.peak2_mean, self.peak_sd, size=Npeak) )

        else:
            Nnormal = Nsamples
            chi_eff, chi_p =  self.model.rvs(size=Nnormal).T

            while True:
                index = np.where((chi_eff>1) + (chi_eff<-1) + (chi_p>1) + (chi_p<0)   )[0]
                n_out = index.size
                if not n_out>0: break
                chi_eff[index], chi_p[index] = self.model.rvs(size=n_out).T

        return chi_eff,chi_p
    def narrow_peak_pdf(self, x):
        xeff, xp = x
        p_xeff = np.exp(-(xeff-self.peak1_mean)**2/(2*self.peak_sd**2))/(np.sqrt(2*np.pi)*self.peak_sd)
        p_xp = np.exp(-(xp-self.peak2_mean)**2/(2*self.peak_sd**2))/(np.sqrt(2*np.pi)*self.peak_sd)
        #print(p_xeff, p_xp)
        return p_xeff * p_xp
    def narrow_peak_rvs(self, mean, sd, size):
        return ss.norm.rvs(mean,sd,size)
    """
    def pdf(self, chi_eff, chi_p):
        if self.peak:
            f = self.lambda_peak
            return  (1-f)*self.model.pdf(np.array([chi_eff,chi_p]).T)/self.normalization[0] + f*self.narrow_peak_pdf(np.array([chi_eff,chi_p]) ) / self.normalization[1]
        else:
            return self.model.pdf(np.array([chi_eff,chi_p]).T) / self.normalization
    """
    def pdf(self, chi_eff, chi_p,x=True):
        if self.peak:
            if x:
                f = self.lambda_peak
            else:
                f = 0
            return  (1-f)*self.model.pdf(np.array([chi_eff,chi_p]).T)/self.normalization[0] + f*self.narrow_peak_pdf(np.array([chi_eff,chi_p]) ) / self.normalization[1]
        else:
            return self.model.pdf(np.array([chi_eff,chi_p]).T) / self.normalization

    def marginal_pxeff(self,x):
        f = self.lambda_peak if self.peak else 0
        p1 = np.exp(-(x-self.mean[0])**2/(2*self.cov[0,0]**2))/(np.sqrt(2*np.pi)*self.cov[0,0])
        p2 = np.exp(-(x-self.peak1_mean)**2/(2*self.peak_sd**2))/(np.sqrt(2*np.pi)*self.peak_sd)

        return (1-f)*p1 + f*p2

    def marginal_pxp(self):
        f = self.lambda_peak if self.peak else 0
        p1 = np.exp(-(x-self.mean[1])**2/(2*self.cov[1,1]**2))/(np.sqrt(2*np.pi)*self.cov[1,1])
        p2 = np.exp(-(x-self.peak1_mean)**2/(2*self.peak_sd**2))/(np.sqrt(2*np.pi)*self.peak_sd)

        return (1-f)*p1 + f*p2

def spin_posterior(chi_eff, chi_p, Nevent, posterior_effsigma = 0.2, posterior_psigma = 0.2, Npos = 1000):
    x_eff = []
    x_p = []
    for i in range(Nevent):
        c1 = TruncNormSampler(-1,1,chi_eff[i],posterior_effsigma,1)
        c2 = TruncNormSampler(0,1,chi_p[i],posterior_effsigma,1)
        x_eff.append(TruncNormSampler(-1,1,c1,posterior_effsigma,Npos))
        x_p.append(TruncNormSampler(0,1,c2,posterior_psigma,Npos))
    return np.array(x_eff).reshape(Nevent, Npos), np.array(x_p).reshape(Nevent, Npos)

if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='Generate population and posterior samples.')
    parser.add_argument("-L", dest = "L", action = 'store_true', help = "Generate lensed population", default = False) # True to turn on the lensing mechanism
    parser.add_argument("-N", type = int, help = "number of population events", default=1986)

    args = parser.parse_args()
    N = args.N # number of binary events

    if args.L:
        savedir = './spin_data_lensed_'+str(N)+'.npz'
    else:
        savedir = './spin_data_unlensed_'+str(N)+'.npz'

    # Initialize the spin population model
    spin_pop = Gaussian_spin_distribution(**spin_pars)

    # Generate the spin catalog
    chi_eff, chi_p = spin_pop.sample(Nsamples=N)

    # generate the posterior
    if args.L: #
        chi_eff = np.concatenate([chi_eff, chi_eff])
        chi_p = np.concatenate([chi_p, chi_p])
        eff_posterior, p_posterior = spin_posterior(chi_eff, chi_p, N*2)
    else:
        eff_posterior, p_posterior = spin_posterior(chi_eff, chi_p, N)

    #np.savez(cdir+'/Mock_Data/spin_data.npz',chi_eff=chi_eff,chi_p=chi_p,eff_posterior=eff_posterior,p_posterior=p_posterior)
    np.savez(savedir,chi_eff=chi_eff,chi_p=chi_p,eff_posterior=eff_posterior,p_posterior=p_posterior)


from functools import wraps
from crossword import solve

def run_solver_test(func):
    @wraps(func)
    def wrapped():
        board, words, result = func()
        assert solve(board, words) == result
    return wrapped

@run_solver_test
def test_simple_1():
    board = [{
        'index': 0,
        'direction': 'down',
        'xcoord': 0,
        'ycoord': 0,
        'length': 2,
    }, {
        'index': 1,
        'direction': 'right',
        'xcoord': 0,
        'ycoord': 0,
        'length': 2,
    }]
    words = {'ab', 'at'}
    result = {0: 'ab', 1: 'at'}

    return board, words, result

@run_solver_test
def test_simple_2():
    board = [{
        'index': 0,
        'direction': 'down',
        'xcoord': 0,
        'ycoord': 0,
        'length': 4,
    }, {
        'index': 1,
        'direction': 'down',
        'xcoord': 2,
        'ycoord': 0,
        'length': 4,
    }, {
        'index': 2,
        'direction': 'right',
        'xcoord': 0,
        'ycoord': 1,
        'length': 4,
    }, {
        'index': 3,
        'direction': 'right',
        'xcoord': 0,
        'ycoord': 3,
        'length': 4,
    }]
    words = {'ccaa', 'baca', 'baaa', 'bbbb'}
    result = {0: 'bbbb', 1: 'ccaa', 2: 'baca', 3: 'baaa'}

    return board, words, result

@run_solver_test
def test_complex_1():
    board = []
    index = 0
    for x, y, length in [
        (0, 0, 8),
        (9, 0, 6),
        (0, 2, 15),
        (0, 4, 10),
        (11, 4, 4),
        (0, 6, 6),
        (7, 6, 8),
        (0, 8, 8,),
        (9, 8, 6),
        (0, 10, 4),
        (5, 10, 10),
        (0, 12, 15),
        (0, 14, 6),
        (7, 14, 8)]:
        board.append({
            'index': index,
            'direction': 'right',
            'xcoord': x,
            'ycoord': y,
            'length': length,
        })
        index += 1

    for x, y, length in [
        (0, 0, 7),
        (0, 8, 7),
        (2, 0, 5),
        (2, 6, 9),
        (4, 0, 10),
        (4, 11, 4),
        (6, 0, 6),
        (6, 7, 6),
        (8, 2, 6),
        (8, 9, 6),
        (10, 0, 4),
        (10, 5, 10),
        (12, 0, 9),
        (12, 10, 5),
        (14, 0, 7),
        (14, 8, 7)]:
        board.append({
            'index': index,
            'direction': 'down',
            'xcoord': x,
            'ycoord': y,
            'length': length,
        })
        index += 1

    words = {
        'AMEN',
        'APPEAL',
        'BANTAM',
        'BLUSTERY',
        'CASTIGATE',
        'CONJUROR',
        'CREASED',
        'CRUCIVERBALISTS',
        'DEMAND',
        'DISPLEASES',
        'DOMAIN',
        'EYES',
        'LARGELY',
        'MANNERISM',
        'MIST',
        'NAVIGATE',
        'NEEDLE',
        'NOMADS',
        'PACESETTER',
        'PENCIL',
        'PICKSON',
        'POINTEDOUT',
        'POSTPONE',
        'PRICER',
        'ROSETTE',
        'RULE',
        'SAUCE',
        'SWEATGLAND',
        'SWIMMINGCOSTUME',
        'TRUCE'
    }
    result = {
        0: 'POSTPONE',
        1: 'PRICER',
        2: 'CRUCIVERBALISTS',
        3: 'SWEATGLAND',
        4: 'MIST',
        5: 'NOMADS',
        6: 'NAVIGATE',
        7: 'CONJUROR',
        8: 'APPEAL',
        9: 'EYES',
        10: 'PACESETTER',
        11: 'SWIMMINGCOSTUME',
        12: 'DEMAND',
        13: 'BLUSTERY',
        14: 'PICKSON',
        15: 'CREASED',
        16: 'SAUCE',
        17: 'MANNERISM',
        18: 'POINTEDOUT',
        19: 'AMEN',
        20: 'NEEDLE',
        21: 'DOMAIN',
        22: 'BANTAM',
        23: 'PENCIL',
        24: 'RULE',
        25: 'DISPLEASES',
        26: 'CASTIGATE',
        27: 'TRUCE',
        28: 'ROSETTE',
        29: 'LARGELY'
    }

    return board, words, result


#!/usr/bin/env python3

import os
os.chdir('..')

import torch
# torch.autograd.set_detect_anomaly(True)
import itertools
from torch.nn.utils.rnn import pad_sequence
from torch.utils.data import DataLoader
from know_how_optimizer.lr_scheduler import WarmupMultiStepLR
from model import distance, Stacked
from model.distance.hyperbolic import HyperbolicHead
from model.encoder import EncoderLSTM
from model.distance import EuclideanHead
from utils.free_group import pairwise_distances
from data import GroupDatasetRandom
import argparse

def collate_wrapper(batch):
    return pad_sequence([
        torch.IntTensor(sen['sequence'][0])
        for sen in batch
    ], batch_first=True, padding_value=0), [sen['sequence'][1] for sen in batch]

dim = 256
generators = 2
group_dataset = GroupDatasetRandom(100, generators)

device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

parser = argparse.ArgumentParser(
    description="Train embeddings with distances for Cayley graph of the free group.")
parser.add_argument(
    "dist", type=str, help="Distance type (euclid, hyp)", default="lstm")
args = parser.parse_args()

encoder = EncoderLSTM(generators, dimension=dim)

if args.dist == "euclid":
    head = EuclideanHead(dimension=dim, lp=2)
elif args.dist == "hyp":
    head = HyperbolicHead(dimension=dim, method="poincare")
else:
    raise AttributeError("Undefined distance notion `" + args.dist + "`")

model = Stacked(encoder, head).to(device)

optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
criterion = torch.nn.MSELoss().to(device)

batch_size = 10
for _ in range(10):
    epoch_loss = 0.0
    iters = 0

    model.train()
    data_loader = itertools.islice(DataLoader(
        group_dataset, batch_size=batch_size,
        shuffle=True, num_workers=0,
        collate_fn=collate_wrapper
    ), batch_size*50)

    for batch in data_loader:
        optimizer.zero_grad()
        sequences, sequences_lengths = batch

        dists = model(sequences, sequences_lengths)
        loss = criterion(
            dists, pairwise_distances(sequences).unsqueeze(0)
        ) / (batch_size**2)
        loss.backward()
        optimizer.step()

        epoch_loss += loss.item()
        iters += 1

    print('loss:', epoch_loss / iters)


from unittest import expectedFailure

from PySide import QtCore, QtGui

from schematics.mouse_modes.line_submode.inserting import (
    GetHightowerObjectAtPoint, LineRouteBetweenPoints, EndpointTrees,
    RouteNotFoundException)
from schematics import GridScene
from algorithms import hightower
from logicitems import LineTree
from tests.helpers import wait_until_registry_enumerated
from tests import helpers

class TestHightowerObject(helpers.CriticalTestCase):
    def setUp(self):
        super().setUp()

        self.app = QtGui.QApplication.instance()
        if not self.app:
            self.app = QtGui.QApplication([])

        self.scene = GridScene()

        # wait until all types have been enumerated
        wait_until_registry_enumerated(self.scene, self.app)

    def tearDown(self):
        # FIXME: No idea why this workaround is necessary :(
        self.scene.deleteLater()
        self.scene._core.quit()
        self.scene._core_thread.join()
        self.scene._registry._registry_handler.quit(True)
        self.scene = None

        self.app.processEvents()

        super().tearDown()

    def test_empty_scene(self):
        trees = EndpointTrees(None, None)
        ho = GetHightowerObjectAtPoint(self.scene, (0, 0), (10, 10), trees)

        self.assertIs(ho((0, 0)), None)
        self.assertIs(ho((10, 0)), None)
        self.assertIs(ho((0, 10)), None)
        self.assertIs(ho((10, 10)), None)

    def test_colision_add_item(self):
        and_item = self.scene.registry().instantiate_frontend_item(
            "7793F2A0-B313-4489-ABF3-8570ECDFE3EE")
        and_item.setPos(self.scene.to_scene_point((2, 2)))
        self.scene.addItem(and_item)
        trees = EndpointTrees(None, None)
        ho = GetHightowerObjectAtPoint(self.scene, (0, 0), (10, 10), trees)

        self.assertIs(ho((0, 0)), None)
        self.assertIs(ho((10, 0)), None)

        self.assertIs(ho((2, 2)), hightower.Solid)

    def test_colision_line_tree(self):
        def tsp(x, y):
            return self.scene.to_scene_point((x, y))
        tree_meta = LineTree.metadata_from_path([tsp(5, 0), tsp(15, 0)])
        tree_meta['GUID'] = LineTree.GUI_GUID()
        tree = LineTree(parent=None, metadata=tree_meta)
        self.scene.addItem(tree)
        trees = EndpointTrees(None, None)
        ho = GetHightowerObjectAtPoint(self.scene, (0, 0), (10, 10), trees)

        self.assertIs(ho((0, 0)), None)
        self.assertIs(ho((5, 5)), None)

        self.assertIs(ho((5, 0)), hightower.LineEdge)
        self.assertIs(ho((15, 0)), hightower.LineEdge)

        self.assertIs(ho((10, 0)), hightower.PassableLine)

class TestLineRoute(helpers.CriticalTestCase):
    def setUp(self):
        super().setUp()

        self.app = QtGui.QApplication.instance()
        if not self.app:
            self.app = QtGui.QApplication([])

        self.scene = GridScene()

        # wait until all types have been enumerated
        wait_until_registry_enumerated(self.scene, self.app)

    def tearDown(self):
        # FIXME: No idea why this workaround is necessary :(
        self.scene.deleteLater()
        self.scene._core.quit()
        self.scene._core_thread.join()
        self.scene._registry._registry_handler.quit(True)
        self.scene = None

        self.app.processEvents()

        super().tearDown()

    def test_horizontal_line(self):
        def tsp(x, y):
            return self.scene.to_scene_point((x, y))
        line_route = LineRouteBetweenPoints(self.scene,
                                            tsp(5, 10),
                                            tsp(15, 10))
        line_route.route()
        line_route.do_temp_insert()

        # check line count & temp
        trees = [item for item in self.scene.items() if isinstance(item,
                                                                   LineTree)]
        self.assertEqual(len(trees), 1)
        self.assertTrue(trees[0].is_temporary())

        # insert & check line count & temp
        line_route.do_insert()
        trees = [item for item in self.scene.items() if isinstance(item,
                                                                   LineTree)]
        self.assertEqual(len(trees), 1)
        self.assertFalse(trees[0].is_temporary())

        # check edges
        tree = trees[0]
        self.assertTrue(tree.is_edge(tsp(5, 10)))
        self.assertTrue(tree.is_edge(tsp(15, 10)))

        # contains line
        self.assertTrue(tree.contains_line(QtCore.QLineF(tsp(5, 10),
                                                         tsp(15, 10))))

        self.assertFalse(tree.contains_line(QtCore.QLineF(tsp(5, 10),
                                                          tsp(4, 10))))
        self.assertFalse(tree.contains_line(QtCore.QLineF(tsp(5, 10),
                                                          tsp(5, 9))))
        self.assertFalse(tree.contains_line(QtCore.QLineF(tsp(5, 10),
                                                          tsp(5, 11))))

class TestLineRouteGraphical(helpers.CriticalTestCase):
    def setUp(self):
        super().setUp()

        self.app = QtGui.QApplication.instance()
        if not self.app:
            self.app = QtGui.QApplication([])

        self.scene = GridScene()

        # wait until all types have been enumerated
        wait_until_registry_enumerated(self.scene, self.app)

    def tearDown(self):
        # FIXME: No idea why this workaround is necessary :(
        self.scene.deleteLater()
        self.scene._core.quit()
        self.scene._core_thread.join()
        self.scene._registry._registry_handler.quit(True)
        self.scene = None

        self.app.processEvents()

        super().tearDown()

    def fill_scene(self, scene, input_area):
        lines = input_area.split('\n')

        def add_tree(p1, p2):
            path = [scene.to_scene_point(p1), scene.to_scene_point(p2)]
            metadata = LineTree.metadata_from_path(path)
            l_tree = self.scene.registry().instantiate_frontend_item(
                backend_guid=LineTree.GUI_GUID(),
                additional_metadata=metadata)
            scene.addItem(l_tree)

        # insert line stubs
        for y, line in enumerate(lines):
            for x, char in enumerate(line):
                if char in '-+┬┴':  # horizontal
                    add_tree((x - 0.5, y), (x + 0.5, y))
                if char in '┤+├|':
                    add_tree((x, y - 0.5), (x, y + 0.5))
                if char in '┐˄┬┌x':
                    add_tree((x, y), (x, y + 0.5))
                if char in '┘└˅┴x':
                    add_tree((x, y), (x, y - 0.5))
                if char in '┌└<├x':
                    add_tree((x, y), (x + 0.5, y))
                if char in '>┤┐┘x':
                    add_tree((x, y), (x - 0.5, y))

        # merge trees
        for y, line in enumerate(lines):
            for x, char in enumerate(line):
                for p in [(x, y), (x + 0.5, y), (x, y + 0.5)]:
                    pivot = scene.to_scene_point(p)
                    items = [item for item in scene.items(pivot)
                             if isinstance(item, LineTree)]
                    merger_tree = None
                    for item in items:
                        if not item.is_edge(pivot):
                            continue
                        if merger_tree is None:
                            merger_tree = item
                    if merger_tree is not None:
                        for item in items:
                            if item is not merger_tree:
                                merger_tree.merge_tree(item)
                                scene.removeItem(item)

    def to_positions(self, scene, position_area):
        lines = position_area.split('\n')

        for y, line in enumerate(lines):
            for x, char in enumerate(line):
                if char in 'AX':
                    point_a = x, y
                if char in 'BX':
                    point_b = x, y

        return (scene.to_scene_point(point_a),
                scene.to_scene_point(point_b))

    def verify_output_data(self, scene, output_area):
        lines = output_area.split('\n')

        for y, line in enumerate(lines):
            for x, char in enumerate(line):
                pivot = scene.to_scene_point((x, y))
                items = [item for item in scene.items(pivot)
                         if isinstance(item, LineTree)]
                if char in 'x┴˄˅<┤├>┬-|┘┐┌└':
                    self.assertEqual(len(items), 1, (x, y))
                    self.assertEqual(items[0].is_edge(pivot),
                                     char in 'x┴˄˅<┤├>┬┘┐┌└')
                elif char in ['+']:
                    self.assertLessEqual(len(items), 2)
                    for item in items:
                        self.assertEqual(item.is_edge(pivot), False)
                else:
                    self.assertEqual(len(items), 0)

    def visual_test(self, input_area, position_area, output_area,
                    view_result=False):
        """
        Visual test case definition for inserting lines.

        Visual test cases are defined based on tree areas, basically
        strings defined as a block representing a two-dimensional grid:

        :param input_area: define lines for the starting scene
        :param position_area: define start and endpoint for inserting
        :param output_area: define output lines

        Lines are encoded with unicode characters: ┴˄˅<┤├>┬-|┘┐┌└:
            - horizontal peace, | vertical mid peace
            < left end, > right end, ˄ upper end, ˅ lower end
            + crossing lines, x intersecting lines
            ┤┬ ┘┐┌└ ┴├, corners

        Positions are given as A, B and X:
            A start, B end, X start and end

        When specifying the areas it is advisable to copy the input area,
        to the position area, then adding positions, then copy the result
        to the ouput area and add the added line.

        :param view_result: open scene with resulting grid for debugging

        """
        self.fill_scene(self.scene, input_area)
        start, end = self.to_positions(self.scene, position_area)
        line_route = LineRouteBetweenPoints(self.scene, start, end)
        try:
            line_route.route()
        except RouteNotFoundException:
            pass
        line_route.do_insert()

        if view_result:
            import schematics.mouse_modes
            import sys
            app = QtGui.QApplication.instance()
            if app is None:
                app = QtGui.QApplication(sys.argv)
            view = schematics.EditSchematicView()
            view.setScene(self.scene)
            view.show()
            view.ensureVisible(QtCore.QRectF(0, 0, 0, 0))
            view.setMouseMode(schematics.mouse_modes.InsertLineMode)
            app.exec_()

        self.verify_output_data(self.scene, output_area)

    def test_horizontal_line(self):
        # ┴˄˅<┤├>┬-|┘┐┌└
        input_area = """

        """
        position_area = """

           A             B

        """
        output_area = """

           <------------->

        """
        self.visual_test(input_area, position_area, output_area)

    def test_simple_split_line(self):
        # ┴˄˅<┤├>┬-|┘┐┌└
        input_area = """
           <----------->
        """
        position_area = """
           <----A------>

                B
        """
        output_area = """
           <----┬------>
                |
                |
                |
                ˅
        """
        self.visual_test(input_area, position_area, output_area)

    def test_simple_crossover_line(self):
        # ┴˄˅<┤├>┬-|┘┐┌└
        input_area = """

           <----------->

        """
        position_area = """
                 A
           <----------->
                 B
        """
        output_area = """
                 ˄
           <-----+----->
                 ˅
        """
        self.visual_test(input_area, position_area, output_area)

    def test_simple_crossover_two_lines(self):
        # ┴˄˅<┤├>┬-|┘┐┌└
        input_area = """

           <----------->
           <----------->
        """
        position_area = """"
                A
           <----------->
           <----B------>
        """
        output_area = """"
                ˄
           <----+------>
           <----┴------>
        """
        self.visual_test(input_area, position_area, output_area)

    def test_simple_connected_crossover_two_lines(self):
        # ┴˄˅<┤├>┬-|┘┐┌└
        input_area = """

           ┌----------->
           └----------->
        """
        position_area = """"
                A
           ┌----------->
           └----B------>
        """
        output_area = """"
                ˄
           ┌----+------>
           └----┴------>
        """
        self.visual_test(input_area, position_area, output_area)

    def test_simple_route_accross_edge(self):
        # ┴˄˅<┤├>┬-|┘┐┌└
        input_area = """
                 ┌----------->
                 |
           ┌-----┴------------>
           └----------->

        """
        position_area = """
                 ┌-----A----->
                 |
           ┌-----┴------------>
           └----------->
                       B
        """
        output_area = """
                 ┌----┬------>
                 |    |
           ┌-----┴----+------->
           └----------+>
                      └>
        """
        self.visual_test(input_area, position_area, output_area)

    def test_simple_connecting_two_lines(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
           ˄        ˄
           |        |
           |        |
           |        |
           ˅        ˅
        """
        position_area = """
           ˄        ˄
           |        |
           A        B
           |        |
           ˅        ˅
        """
        output_area = """
           ˄        ˄
           |        |
           ├--------┤
           |        |
           ˅        ˅
        """
        self.visual_test(input_area, position_area, output_area)

    def test_extend_over_corner(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
           <--------┐
                    |
                    |
                    |
                    ˅
        """
        position_area = """
           <---A----┐    B
                    |
                    |
                    |
                    ˅
        """
        output_area = """
           <--------┬---->
                    |
                    |
                    |
                    ˅
        """
        self.visual_test(input_area, position_area, output_area)

    def test_extend_over_corner_swapped(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
           <--------┐
                    |
                    |
                    |
                    ˅
        """
        position_area = """
           <---B----┐    A
                    |
                    |
                    |
                    ˅
        """
        output_area = """
           <--------┬---->
                    |
                    |
                    |
                    ˅
        """
        self.visual_test(input_area, position_area, output_area)

    def test_extend_over_two_corners(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
           <--------┬-----┐
                    |     |
                    |     |
                    |     |
                    ˅     ˅
        """
        position_area = """
           <---A----┬-----┐     B
                    |     |
                    |     |
                    |     |
                    ˅     ˅
        """
        output_area = """
           <--------┬-----┬----->
                    |     |
                    |     |
                    |     |
                    ˅     ˅
        """
        self.visual_test(input_area, position_area, output_area)

    def test_connect_passing_crossing_lines(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
              ˄
              |
           <--+-->
              |
              ˅
        """
        position_area = """
              ˄
              |
           <--X-->
              |
              ˅
        """
        output_area = """
              ˄
              |
           <--x-->
              |
              ˅
        """
        self.visual_test(input_area, position_area, output_area)

    def test_try_route_from_passing_crossing_lines(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
              ˄
              |
           <--+-->
              |
              ˅
        """
        position_area = """
              ˄
              |
        B  <--A-->
              |
              ˅
        """
        output_area = """
              ˄
              |
           <--+-->
              |
              ˅
        """
        self.visual_test(input_area, position_area, output_area)

    def test_simple_route_accross_gap(self):
        # ┴˄˅<┤├>┬-|┘┐┌└
        input_area = """

           <------┐┌----->
                  └┘
        """
        position_area = """

           <---A--┐┌----->    B
                  └┘
        """
        output_area = """
               ┌--------------┐
           <---┴--┐┌----->    ˅
                  └┘
        """
        self.visual_test(input_area, position_area, output_area)

    def test_try_route_loop(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
            ┌-----┐
            |     |
            |     |
            |     |
            ˅     ˅
        """
        position_area = """
            ┌-----┐
            |     |
            A     B
            |     |
            ˅     ˅
        """
        output_area = """
            ┌-----┐
            |     |
            |     |
            |     |
            ˅     ˅
        """
        self.visual_test(input_area, position_area, output_area)

    @expectedFailure
    def test_verifyer(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
            <----->
        """
        position_area = """
            A-----B
        """
        output_area = """
                 ->
        """
        self.visual_test(input_area, position_area, output_area)

    def test_insert_over_edge(self):
        # ┴˄˅<┤├>┬-|┘┐┌└|
        input_area = """
            <----->
        """
        position_area = """
            <----->    B

                A
        """
        output_area = """
            <----->    ˄
                       |
                <------┘
        """
        self.visual_test(input_area, position_area, output_area)