import pygame
from pygame.locals import *
from pygame.math import Vector2
import pytmx
import utils

class Door(pygame.sprite.Sprite):
    def __init__(self, x, y, world):
        super().__init__()
        self.state = 'CLOSED'
        self.anim_frame = 0
        self.world = world
        self.sprites = utils.load_door_sprites()
        self.image = self.sprites[0]
        self.rect = self.image.get_rect()
        self.rect.x = x
        self.rect.y = y
        self.rect.width = self.image.get_width()
        self.rect.height = self.image.get_height()
        print(f"Door created at {x}, {y} with size {self.rect.width}, {self.rect.height}")

    def toggle(self):
        """Toggles the door state.
            - If the door is closed, it will start opening.
            - If the door is open, it will start closing.
        """
        if self.state == 'CLOSED':
            self.open()
        elif self.state == 'OPEN':
            self.close()
        else:
            print(f"Door cannot be toggled from {self.state} state")

    def open(self):
        """Opens the door.
            - If the door is closed, it will start opening.
        """
        if self.state == 'CLOSED':
            self.state = 'OPENING'
            self.anim_frame = 0
        else:
            print(f"Door cannot be opened from {self.state} state")

    def close(self):
        """Closes the door.
            - If the door is open, it will start closing.
        """
        if self.state == 'OPEN':
            self.state = 'CLOSING'
        else:
            print(f"Door cannot be closed from {self.state} state")

    def is_open(self):
        return self.state == 'OPEN'

    def view(self):
        return self.sprites[self.anim_frame]

    def update(self):
        if self.state == 'OPENING':
            self.anim_frame += 1
            if self.anim_frame == len(self.sprites) - 1:
                self.state = 'OPEN'
        elif self.state == 'CLOSING':
            self.anim_frame -= 1
            if self.anim_frame == 0:
                self.state = 'CLOSED'

class World:
    def __init__(self, tiled_map):
        """
        Represents the game world.
        tiled_map: pytmx.TiledMap, the Tiled map data
        """
        self.tiled_map = tiled_map
        self.tilesize = tiled_map.tilewidth
        self.width = self.tiled_map.width
        self.height = self.tiled_map.height
        self.ground = self.tiled_map.get_layer_by_name('Ground')
        self.foreground = self.tiled_map.get_layer_by_name('Foreground')
        self.objects = self.tiled_map.get_layer_by_name('Objects')
        self.doors = {(d.x // self.tilesize, d.y // self.tilesize): Door(d.x, d.y, self) for d in self.tiled_map.get_layer_by_name('Doors')}

    def update(self):
        for door in self.doors.values():
            door.update()

    def will_collide(self, x,y):
        """
            Returns true if the player will collide with the world at the given position.
        """
        if x < 0 or x >= self.width or y < 0 or y >= self.height:
            return True
        if self.objects.data[y][x] != 0:
            return True
        if (x,y) in self.doors and not self.doors[(x,y)].is_open():
            return True
        return False

    def draw_ground(self, surface, debug=False):
        """Blits the ground layer to the surface.
            surface: pygame.Surface, must be (self.width * self.tilesize, self.height * self.tilesize)
        """
        for x, y, gid, in self.ground:
            tile = self.tiled_map.get_tile_image_by_gid(gid)
            if tile:
                tile.set_colorkey((0, 0, 0))
                surface.blit(tile, (x * self.tilesize, y * self.tilesize))
        return surface

    def draw_objects(self, surface: pygame.Surface, debug=False):
        """Blits the objects layer to the surface.
            surface: pygame.Surface, must be (self.width * self.tilesize, self.height * self.tilesize)
        """
        for x, y, gid, in self.objects:
            tile = self.tiled_map.get_tile_image_by_gid(gid)
            if tile:
                tile.set_colorkey((0, 0, 0))
                surface.blit(tile, (x * self.tilesize, y*self.tilesize))
                if debug:
                    # create a red rectangle around the object
                    rect = pygame.Rect(x * self.tilesize, y * self.tilesize, self.tilesize, self.tilesize)
                    # draw the outline of the rect in red on the surface
                    pygame.draw.rect(surface, (255, 0, 0), rect, 1)

        for door in self.doors.values():
            surface.blit(door.view(), door.rect)

        return surface

    def draw_foreground(self, surface: pygame.Surface, debug=False):
        """Blits the foreground layer to the surface.
            surface: pygame.Surface, must be (self.width * self.tilesize, self.height * self.tilesize)
        """
        for x, y, gid, in self.foreground:
            tile = self.tiled_map.get_tile_image_by_gid(gid)
            if tile:
                tile.set_colorkey((0, 0, 0))
                surface.blit(tile, (x * self.tilesize, y*self.tilesize))
                rect = pygame.Rect(x * self.tilesize, y * self.tilesize, self.tilesize, self.tilesize)
        return surface

class Player(pygame.sprite.Sprite):
    def __init__(self, x, y, world: World, sprites):
        """
            Represents player, the main controllable character.
            x: the x position of the player in tiles
            y: the y position of the player in tiles
            world: the world object
            sprites:
                a dictionary of sprites for the player, where the key is a direction and the value is a
                list containing pygame.Surface instances
                Example: {'DOWN': [pygame.Surface, pygame.Surface, pygame.Surface, pygame.Surface]}
        """
        pygame.sprite.Sprite.__init__(self)
        self.x = x
        self.y = y
        self.world = world
        self.sprites = sprites
        self.state = 'IDLE'
        self.direction = 'DOWN'
        self.anim_frame = 0
        self.speed = 1 # 1 tile per move
        self.image = self.sprites[self.direction][self.anim_frame]
        self.rect = self.image.get_rect()
        self.tick = 0

    def movemap(self, direction):
        """
            Returns the x and y offset for the given direction.
        """
        mmap = {
                'UP': (0, -self.speed),
                'DOWN': (0, self.speed),
                'LEFT': (-self.speed, 0),
                'RIGHT': (self.speed, 0),
            }
        return mmap[direction]

    def update(self, actions):
        """
            The update function is called every frame.
            actions: a dictionary of actions that the player has taken.
             Example: {'direction': 'UP'}
        """
        self.tick += 1

        ax,ay = self.movemap(self.direction)
        if (self.x+ax, self.y+ay) in self.world.doors:
            print(self.world.doors[(self.x+ax, self.y+ay)])
            # if key 'SPACE' is pressed, open the door
            # use pygame key constants for this
            if pygame.key.get_pressed()[pygame.K_SPACE]:
                self.world.doors[(self.x+ax, self.y+ay)].toggle()
                return

        if self.state == 'IDLE':
            if actions['direction'] == self.direction:
                dx,dy = self.movemap(actions['direction'])
                if self.world.will_collide(self.x+dx, self.y+dy):
                    return

                # enter the walk state
                self.state = 'WALK'
            elif actions['direction']:
                # face the direction of the keypress
                self.direction = actions['direction']
        elif self.state == 'WALK':
            if self.tick % 2 != 0:
                return
            dx,dy = self.movemap(self.direction)
            # play the walking animation by 1 tick
            self.anim_frame = (self.anim_frame + 1) % len(self.sprites[self.direction])
            if self.anim_frame % 2 == 0: # next frame is standing still, meaning we've entered new tile
                self.x += dx
                self.y += dy
                # then we reset the counter and go back to idle
                self.state = 'IDLE'

            # shift the sprite's position by half a tile each animation frame
            self.rect.x += dx * self.world.tilesize / 2
            self.rect.y += dy * self.world.tilesize / 2

        self.image = self.sprites[self.direction][self.anim_frame]

    def view(self):
        # print(f"Player view: {self.direction}, {self.anim_frame}")
        return self.sprites[self.direction][self.anim_frame]

class Camera:
    def __init__(self, viewport_size, world: World, player: Player, scale=3, debug=False):
        """
            Creates a camera that follows the player around the world.
            viewport_size: the size of the viewport in tiles (W, H)
            world: the world object
            player: the player object
            scale: scale factor for tiles, for example 3 means world.tilesize*3 pixels per tile
        """
        self.scale = scale
        self.world = world
        self.player = player
        self.vw, self.vh = viewport_size[0] * world.tilesize, viewport_size[1] * world.tilesize
        # these are pixel coordinates, whereas every other class's coordinates are tile coordinates
        self.x = 0
        self.y = 0
        self.debug = debug

    def set_debug(self, debug):
        self.debug = debug

    def contains(self, x,y):
        """
            Returns True if the given pixel coordinates are within the camera's viewport.
        """
        return (x >= self.x and x <= (self.x + self.viewportWidth) and
                y >= self.y and y <= (self.y + self.viewportHeight))

    def update(self):
        """
            Updates the camera's position to follow the player.
        """
        # target position should be (px, py) - (viewportWidth/2, viewportHeight/2)
        self.x = self.player.rect.x - self.vw * 0.5
        self.y = self.player.rect.y - self.vh * 0.5
        # self.x = utils.clamp(self.x, 0, self.world.width * self.world.tilesize - self.vw)
        # self.y = utils.clamp(self.y, 0, self.world.height * self.world.tilesize - self.vh)

    def draw(self, surface):
        """
            Draws the contents of the camera's viewport to the given surface.
        """

        # first we draw the entire world to a buffer
        # TODO: only blit the tiles that are visible in the viewport to reduce draw calls
        worldbuffer = pygame.Surface((self.world.width * self.world.tilesize, self.world.height * self.world.tilesize))

        # clear the buffer, then draw the world layer by layer
        worldbuffer.fill((0,0,0))
        self.world.draw_ground(worldbuffer, debug=self.debug)
        self.world.draw_objects(worldbuffer, debug=self.debug)
        worldbuffer.blit(self.player.view(), (self.player.rect.x, self.player.rect.y - self.player.rect.height/2))
        self.world.draw_foreground(worldbuffer, debug=self.debug)

        cambuffer = pygame.Surface((self.vw, self.vh))
        cambuffer.blit(worldbuffer, (0,0), (self.x, self.y, self.vw, self.vh))
        # we subtract the tilesize from the player's y coordinate because the player's y coordinate is the top of the sprite
        # but we want to draw the player's sprite at the bottom of the tile
        # cambuffer.blit(self.player.view(), (self.player.rect.x - self.x, self.player.rect.y - self.y - self.world.tilesize))

        if self.debug:
            # draw text
            pygame.draw.rect(cambuffer, (255,0,255), (self.player.rect.x - self.x, self.player.rect.y - self.y - self.player.rect.height/2, self.player.rect.width, self.player.rect.height), 1)
            pygame.draw.rect(cambuffer, (255,0,0), (self.player.x * self.world.tilesize - self.x, self.player.y * self.world.tilesize - self.y, self.world.tilesize, self.world.tilesize), 1)

        # scale the camera buffer by the scale factor
        scene = pygame.transform.scale(cambuffer, (self.vw * self.scale, self.vh * self.scale))
        if self.debug:
            font = pygame.font.SysFont('Arial', 16)
            text = font.render(f"Player: {self.player.x}, {self.player.y}", True, (255,255,255))
            scene.blit(text, (0,0))

        if self.debug:
            # show which tile the mouse is on
            mx, my = pygame.mouse.get_pos()
            mx = int((mx / self.scale + self.x) / self.world.tilesize)
            my = int((my / self.scale + self.y) / self.world.tilesize)
            text = font.render(f"Mouse: {mx}, {my}", True, (255,255,255))
            scene.blit(text, (0,16))

        # blit the final scene to the surface
        surface.blit(scene, (0,0))

class Game:
    def __init__(self, screen):
        """
            Creates a new game instance.
            screen: the pygame display surface

        """
        self.__setup_window()
        self.screen = screen
        self.screen_dims = screen.get_size()
        print(f"Starting new game with resolution: {self.screen_dims}")
        self.states = ['MENU', 'PLAY', 'EXIT']
        self.game_map = pytmx.load_pygame('tiled/boolground.tmx')
        self.world: World = World(self.game_map)
        self.player: Player = Player(0, 0, self.world, sprites=utils.load_player_sprites())
        camera_dims = self.screen_dims[0] / (self.world.tilesize * 2), self.screen_dims[1] / (self.world.tilesize * 2)
        self.debug = False
        self.camera: Camera = Camera(camera_dims, self.world, self.player, scale=2)
        self.state = 'PLAY'
        self.clock = pygame.time.Clock()
        self.direction_keys = {
            'UP': [pygame.K_UP, pygame.K_w],
            'DOWN': [pygame.K_DOWN, pygame.K_s],
            'LEFT': [pygame.K_LEFT, pygame.K_a],
            'RIGHT': [pygame.K_RIGHT, pygame.K_d],
        }
        self.action_keys = {
            'EXIT': [pygame.K_ESCAPE, pygame.K_q],
            'SPACE': [pygame.K_SPACE],
        }
        self.actions = {
            'direction': None,
            'action': None,
        }

    def __setup_window(self):
        # function starts with __ because it's not meant to be called externally
        """ Configures the window settings """
        pygame.display.set_caption('Bool')
        # pygame.display.set_icon(pygame.image.load('sprites/boolground.png'))
        pygame.display.flip()

    def mainloop(self):
        """
        Main game loop.
        """
        while self.state != 'EXIT':
            # print(pygame.mouse.get_pos(), pygame.mouse.get_rel())
            self.clock.tick(30)
            self.screen.fill((88,88,88))
            self.__handle_events()
            if self.state == 'PLAY':
                self.play()
            elif self.state == 'MENU':
                self.menu()

            pygame.display.flip()
        pygame.display.quit()
        pygame.quit()

    def __handle_events(self):
        """
            Processes events, such as key presses and mouse clicks.
            Updates the game state and stores the actions in the actions dictionary.
        """
        for event in pygame.event.get():
            if event.type == pygame.QUIT:
                self.state = 'EXIT'
            elif event.type == pygame.KEYDOWN:
                # check if 'I' was pressed
                if event.key == pygame.K_i:
                    self.camera.set_debug(not self.camera.debug)

                for direction in self.direction_keys:
                    if event.key in self.direction_keys[direction]:
                        self.actions['direction'] = direction
            elif event.type == pygame.KEYUP:
                # this is a hack to make sure the player stops moving when the key is released
                # it prob doesn't work if the player is holding down two keys at once
                self.actions = {
                'direction': None,
                'action': None,
                }

    def menu(self):
        """
            Updates the menu state and draws to the screen. Should be called every frame.
        """
        pass

    def play(self):
        """
            Updates the game state and draws the game to the screen. Should be called every frame.
        """
        self.player.update(self.actions)
        self.world.update()
        self.camera.update()
        self.camera.draw(self.screen)

SCREEN_WIDTH = 400
SCREEN_HEIGHT = 300
dims = (SCREEN_WIDTH, SCREEN_HEIGHT)

if __name__ == '__main__':
    pygame.init()
    screen = pygame.display.set_mode(dims,flags=pygame.SCALED, vsync=0)
    game = Game(screen)
    game.mainloop()


import sys
import traceback

from PyQt5.QtWidgets import QApplication

from ru.stepchenkov.trace.window.MainWindow import MainWindow

"""
    Ловим ошибки.
"""

def exception_hook(exc_type, exc_value, exc_tb) -> None:
    tb = "".join(traceback.format_exception(exc_type, exc_value, exc_tb))
    print("Err: ", tb)

"""
    Запуск.
"""
if __name__ == '__main__':
    sys.excepthook = exception_hook
    app = QApplication(sys.argv)
    ex = MainWindow()
    app.exec_()
    del ex, app

def start():
    sys.excepthook = exception_hook
    app = QApplication(sys.argv)
    ex = MainWindow()
    app.exec_()
    del ex, app


import json
import nltk
import pdb
from sklearn.metrics.pairwise import cosine_similarity
import time
from tqdm import tqdm
dict_file = '../data/wordVectors/word_dict.json'  #词向量json文件
# attack_data_file = 'data/wikipassageQA/sample_data.json'
attack_data_file = 'data/wikipassageQA/attack_data_allwords.json'
from transformers import BertTokenizer
from nltk.corpus import stopwords
from utils.wordnettools import WNtools
#nltk.download('stopwords')

wntools = WNtools()

# 返回一个dict，key为单词，value为其embedding
# 暂时人为判断替换词应该在0.6以上
def CounterFittedVectors(dict_file):
    all_words = {}
    with open(dict_file, 'r', encoding='utf-8') as reader:
        all_data = reader.read()
        word_dict = json.loads(all_data)
        for dict in word_dict:
            all_words.update(dict)
    return all_words

#读入数据
def getAttackData(attack_data_file):
    all_data = []
    with open(attack_data_file, 'r', encoding='utf-8') as reader:
        lines = reader.readlines()
        for line in lines:
            a_data = json.loads(line)
            all_data.append(a_data)

    return all_data

#检查是否为 后缀 or 标点符号 or 停用词
def checkwords(aword, stopwords):
    punctuate = '[\s+\.\!\/_,$%^*(+\"\']+|[+——！，。?、~@#￥%……&*（）]+'
    if ('#' in aword) or (aword in punctuate) or (aword in stopwords):
        return False
    else:
        return True

#计算两个词在counter-fitted embedding中的余弦相似度
def cal_cos_sim(word1, word2, word_dict):
    embedding1 = word_dict.get(word1, None)
    embedding2 = word_dict.get(word2, None)
    if embedding1 and embedding2:
        results = cosine_similarity([embedding1, embedding2])[0][1]
        return results
    else:
        return None

def attack_a_data(a_data, tokenizer, pos_checking=False):
    sorts = a_data['words_sort']
    positions = a_data['position']
    input_ids = a_data['input_ids']
    result = tokenizer.convert_ids_to_tokens(sorts)
    filter = []
    for word, position in zip(result, positions):
        if checkwords(word, stopwords):
            filter.append((word, position))
    #print(filter)
    # print(len(filter))
    changesNum = int(round(len(filter) * change_porpotion, 0))
    for i in range(changesNum):
        a_pair = filter[i]
        origin_word, position = a_pair
        #origin_pos = nltk.pos_tag([origin_word])[0][1]
        #print(origin_pos)
        if not word_dict.get(origin_word, None):  # 如果词表里没有origin_word就直接跳到下一个
            continue
        if pos_checking:
            rep_list = wntools.GenerateSimWordByWNAndPosChecking(origin_word)
        else:
            rep_list = wntools.GenerateSimWordByWN(origin_word)
                #这里决定是否使用PosTagging
        if not rep_list:  # 如果WordNet没有找到同义词就跳到下一个
            continue
        else:
            results = []
            for rep_word in rep_list:
                sim_scores = cal_cos_sim(origin_word, rep_word, word_dict)
                if (not sim_scores) or (sim_scores < 0.6):
                    continue
                results.append((rep_word, sim_scores))
            if results:  # 从结果中找到符合要求的，替换原来的input_ids即可
                results.sort(reverse=True, key=lambda x: x[1])
                # print('position:', position)
                for result in results:
                    attack_word = result[0]

                    attack_word_idx = tokenizer.convert_tokens_to_ids(attack_word)  #遍历能在tokenizer词表里找到的单词
                    if attack_word_idx != 100:
                        input_ids[position] = attack_word_idx    # 更改对应位置的input_ids并推出循环
                        break
                    else:
                        continue

    a_data['input_ids'] = input_ids
    return a_data

if __name__ == '__main__':
    change_porpotion = 0.3
    start_time = time.time()
    #加载需要的东西
    word_dict = CounterFittedVectors(dict_file)
    all_data = getAttackData(attack_data_file)
    tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
    stopwords = stopwords.words('english')

    pos_checking = True  #是否使用poschecking
    #print(stopwords)
    print('loadingTime:', time.time()-start_time)

    attacked_results = []
    for a_data in tqdm(all_data):
        attacked_data = attack_a_data(a_data, tokenizer, pos_checking=pos_checking)
        attacked_results.append(attacked_data)

    with open('data/wikipassageQA/attacked_rate03_pos.json', 'w', encoding='utf-8') as writer:
        for a_data in tqdm(attacked_results):
            json.dump(a_data, writer)
            writer.write('\n')

    # #pdb.set_trace()
    # start_time = time.time()
    # word1 = 'evangelical'
    # word2 = 'forgiveness'
    # results = cal_cos_sim(word1, word2, word_dict)
    # if results:
    #     print(results)
    # else:
    #     print('词表查无此词')
    # print('CalSimTime:', time.time()-start_time)


import requests
import redis

from bs4 import BeautifulSoup
from datetime import datetime

class Download:
    '下载类'

    def __init__(self, dict_header=None, int_retries=3, int_delay=2, int_timeout=30):
        '''【初始化】
        dict_header：标头
        int_retries：重试
        delay：延迟
        timeout：超时'''
        self.dict_header = dict_header
        self.int_retries = int_retries
        self.int_delay = int_delay
        self.int_timeout = int_timeout

    def download_all(self, str_url):
        '''【下载页面】
        str_url：地址
        bool_json：是否json类型'''
        try:
            response_obj = requests.get(str_url, headers=self.dict_header, timeout=self.int_timeout)
            if response_obj.status_code == 200:
                return response_obj.content
        except Exception as e:
            str_html = ''
        return str_html

class Crawler:
    '爬虫类'

    def __init__(self, dict_header=None, int_retries=3, int_delay=2, int_timeout=30):
        '''【初始化】
        dict_header：标头
        int_retries：重试
        delay：延迟
        timeout：超时'''
        self.dict_header = dict_header
        self.int_retries = int_retries
        self.int_delay = int_delay
        self.int_timeout = int_timeout
        self.download = Download(dict_header, int_retries, int_delay, int_timeout)
        self.redis = Redis()

    def crawler_operate(self, str_url, int_page_start, int_page_end, int_page_step):
        str_date = ''
        int_page_num = 1
        for int_page in range(int_page_start, int_page_end, int_page_step):
            str_date += self.crawler_data(str_url.format(int_page))
            print('第%i页'.center(30, '*') % int_page_num)
            int_page_num += 1
        self.redis.redis_save(str_date)

    def crawler_data(self, str_url):
        '【处理数据】，str_url：地址'
        str_page = self.download.download_all(str_url)
        # 解析内容
        soup_obj = BeautifulSoup(str_page, 'lxml')
        soup_all_items = soup_obj.find_all('div', attrs={'class': 't_l_bd'})
        str_date = ''
        for soup_item in soup_all_items:
            # print('='*30)
            str_en = soup_item.find('p', attrs={'class': 't_l_en'}).find('a').text
            str_cn = soup_item.find('p', attrs={'class': 't_l_cn'}).find('a').text
            str_date += ('{}{}_'.format(str_en, str_cn))
        return str_date

    # def crawler_re_chars(self, str_text):
    #     '【过滤特殊符号】，str_text：文本'
    #     return re.sub('[^\u4e00-\u9fa5a-zA-Z0-9\'\",.，。!@#$%&*()]', '', str_text)

class Redis:

    def __init__(self, str_host='127.0.0.1', int_port=6379):
        '【初始化】'
        self.obj_clinet = redis.StrictRedis(str_host, int_port)

    def redis_save(self, str_value):
        '【记录】'
        self.obj_clinet.set(datetime.now().strftime("%Y%m%d%H%M"), str_value)

    def redis_query(self, str_key, str_count):
        '【查询】'
        try:
            list_value = self.obj_clinet.get(str_key).decode().split('_')
            for i in range(0, int(str_count)):
                print(list_value[i])
        except Exception as e:
            return 0

def main():
    # 地址
    str_url = 'http://news.iciba.com/appv3/wwwroot/ds.php?action=tags&id=4&ob=1&order=2&page={}'

    # 标头
    dict_header = {
        'User-agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/57.0.2987.133 Safari/537.36',
        'referer': 'http://my.iciba.com/index.php'
    }
    # 实例化
    crawler_obj = Crawler(dict_header=dict_header)
    # 获取3页数据
    # crawler_obj.crawler_operate(str_url, 1, 4, 1)
    Redis().redis_query('201810272251', 6)

if __name__ == '__main__':
    main()


import random

import pygame as pg

# CONSTANTS
SCREEN_WIDTH = 1200
SCREEN_HEIGHT = 800
FPS = 10000

class Barnsley:
    """Make an object that draws a fern."""

    def __init__(self):
        self.x = 0
        self.y = 0
        self.xprint = None
        self.yprint = None
        self.R = 0
        self.G = 0
        self.B = 0
        self.colX = 0
        self.colY = 0

    def make_point(self):
        r = random.randrange(0, 100, 1)
        if r < 1:
            self.x = 0
            self.y = 0.16 * self.y
        elif r < 86:
            self.x = 0.85 * self.x + 0.04 * self.y
            self.y = -0.04 * self.x + 0.85 * self.y + 1.6
        elif r < 93:
            self.x = 0.2 * self.x - 0.26 * self.y
            self.y = 0.23 * self.x + 0.22 * self.y + 1.6
        else:
            self.x = -0.15 * self.x + 0.28 * self.y
            self.y = 0.26 * self.x + 0.24 * self.y + 0.44

    def draw(self, screen):
        # −2.1820 < x < 2.6558
        # 0 ≤ y < 9.9983
        self.xprint = int(self.x * SCREEN_WIDTH * 0.1) + int(SCREEN_WIDTH * 0.45)
        self.yprint = -int(self.y * SCREEN_HEIGHT * 0.07) + int(SCREEN_HEIGHT * 0.85)
        self.colX = (self.x + 2.2) / 5.1
        self.colY = self.y / 12
        self.R = int((self.colY) * 255)
        self.G = int((self.colX) * 255)
        self.B = int((2 - self.colX - self.colY) * 255 / 2)
        screen.set_at((self.xprint, self.yprint), (self.R, self.G, self.B))

class Sim:
    """Simulate the making of the fern."""

    def __init__(self):
        pg.init()
        self.FPS = FPS
        self.screen_width = SCREEN_WIDTH
        self.screen_height = SCREEN_HEIGHT
        self.screen = pg.display.set_mode((self.screen_width, self.screen_height))
        pg.display.set_caption("Barnsley Fern")
        self.clock = pg.time.Clock()
        self.setup()
        self.a = 0

    def setup(self):
        self.fern = Barnsley()
        self.screen.fill((20, 2, 2))

    def events(self):
        for event in pg.event.get():
            if event.type == pg.KEYDOWN:
                if event.key == pg.K_r:
                    self.setup()
                if event.key == pg.K_ESCAPE:
                    # pg.image.save(self.screen, 'barnsley_fern.png')
                    pg.quit()
                    quit()
            if event.type == pg.QUIT:
                pg.quit()
                quit()

    def sim_loop(self):
        while 1:
            self.fern.draw(self.screen)
            self.fern.make_point()
            # pg.display.flip()
            if self.a % 500 == 0:
                pg.display.update()
            # pg.transform.flip(self.screen, False, True)
            self.clock.tick(self.FPS)
            self.events()
            self.a += 1

if __name__ == "__main__":
    Sim().sim_loop()


""" Language Repository module """

from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session

from app.languages.exceptions import LanguageNotFoundException
from app.languages.models import Language

class LanguageRepository:
    """Language model repository"""

    def __init__(self, db: Session):
        self.db = db

    ##superuser
    def create_language(self, name, abbreviation):
        """Create new language"""
        try:
            language = Language(name, abbreviation)
            self.db.add(language)
            self.db.commit()
            self.db.refresh(language)
            return language
        except IntegrityError as e:
            raise e from e

    def get_language_by_id(self, language_id: str):
        """Get language by id"""
        language = self.db.query(Language).filter(Language.id == language_id).first()
        if language is None:
            raise LanguageNotFoundException(
                f"Language with provided id: {language_id} not found.", 400
            )
        return language

    def get_language_by_name(self, name: str):
        """Get language by category"""
        language = (
            self.db.query(Language).filter(Language.name.ilike(f"%{name}%")).all()
        )
        if (language is None) or (language == []):
            raise LanguageNotFoundException(
                f"Language with provided category: {name} not found.", 400
            )
        return language

    def get_language_by_abbreviation(self, abbreviation: str):
        """Get language by abbreviation"""
        language = (
            self.db.query(Language)
            .filter(Language.abbreviation.ilike(f"%{abbreviation}%"))
            .all()
        )
        if (language is None) or (language == []):
            raise LanguageNotFoundException(
                f"Language with provided abbreviation: {abbreviation} not found.", 400
            )
        return language

    def get_all_languages(self):
        """Get all languages"""
        languages = self.db.query(Language).all()
        if (languages is None) or (languages == []):
            raise LanguageNotFoundException(f"The list is empty!", 400)
        return languages

    ##superuser
    def delete_language_by_id(self, language_id: str):
        """Delete language by id"""
        try:
            language = (
                self.db.query(Language).filter(Language.id == language_id).first()
            )
            if language is None:
                raise LanguageNotFoundException(
                    f"Language with provided id: {language_id} not found.", 400
                )
            self.db.delete(language)
            self.db.commit()
            return True
        except Exception as e:
            raise e from e


"""A matchmaking service for GZM.

Uses an airtable list of interested people to create matches at random.
"""
import random
import os
from typing import List, Literal, TypedDict

import airtable
import discord

TOKEN = os.getenv("DISCORD_TOKEN")
AIRTABLE_BASE = os.getenv("AIRTABLE_BASE")
AIRTABLE_API_KEY = os.getenv("AIRTABLE_API_KEY")

client = discord.Client()
airtable = airtable.Airtable(AIRTABLE_BASE, "dumble", AIRTABLE_API_KEY)

copy = {
    "intro": "Welcome to Dumble! Are you open to matching in a romantic or friendly manner today?",
    "friendly": "ðŸ˜ƒ",  # smiley face emoji
    "romantic": "ðŸ’•",  # two hearts emoji
    "either": "ðŸ˜Š",  # blushing emoji
}

romanticPool = set()  # {discord.User}, remove user when matched
friendlyPool = set()

UserFields = TypedDict(
    "UserFields",
    {
        "Email": str,
        "Twitter": str,
        "Name": str,
        "Phone": str,
        "Discord": str,
        "I am a": Literal["Man", "Woman", "Other"],
        "Looking for": List,
    },
)

class User(TypedDict):
    id: str
    fields: UserFields
    createdTime: str

class DiscordUserNotFound(Exception):
    pass

def list_available_users() -> List[User]:
    """List the available users in our airtable DB.

    Todo:
        - Only get the fields that I need:
    https://airtable-python-wrapper.readthedocs.io/en/master/params.html

    """
    return airtable.get_all(formula="AND(NOT({Email}=''), NOT({Looking for}=''))")

def find_match(user: User) -> User:
    """Find a match for a given user."""

    all_users = list_available_users()
    looking_for = user["fields"]["Looking for"]  # ['Man', 'Woman', ...]

    potential_matches = [
        user for user in all_users if user["fields"]["I am a:"] in looking_for
    ]

    return random.choice(potential_matches)

def find_match_by_discord_id(discord_id: str) -> (User, User):
    """Find match by discord id, so we can build a discord bot that matches people.

    discord_id should be of the form: "JaneDoe#1234"
    """
    users = airtable.search("Discord", discord_id)

    if len(users) == 0:
        raise DiscordUserNotFound("Could not find any users with that id.")
    user = users[0]  # if multiple users are returned, just grab the first one Â¯\_(ãƒ„)_/Â¯
    match = find_match(user)

    return user, match

# link to database: https://airtable.com/tbl6wkasPz9DHFdbp/viwMq7Xv2hRIzuZlb?blocks=hide

@client.event
async def on_ready():
    """on_ready is called when the discord bot first loads."""
    print(f"{client.user} has connected to Discord!")

async def addReactions(msg: discord.Message, emojis: List):
    for emoji in emojis:
        await msg.add_reaction(emoji)

async def userInterface(user: discord.Member, channel):
    introMsg = None
    try:
        introMsg = await user.send(copy["intro"])
    except discord.Forbidden:
        await channel.send(
            user.display_name
            + " please allow DM's from this bot and retype '!dumble' in order to proceed!"
        )
        return

    emojis = [copy["romantic"], copy["friendly"]]
    await addReactions(introMsg, emojis)

@client.event
async def on_reaction_add(reaction: discord.Reaction, user: discord.User):
    # Romantic Or Friendly
    if (
        user != client.user
        and reaction.message.author == client.user
        and reaction.message.content == copy["intro"]
    ):
        print("Reaction added to bot DM!", str(reaction))
        if str(reaction) == copy["romantic"]:
            romanticPool.add(user)
            print(romanticPool)
        elif str(reaction) == copy["friendly"]:
            friendlyPool.add(user)
            print(friendlyPool)
    # Match Or Don't Match
    elif ():
        pass

    # Base Case
    else:
        return

def getUserData(discord_id: str):
    """
    Find user in airtable else return None
    """
    users = airtable.search("Discord", discord_id)

    if len(users) == 0:
        return None
    return users[0]  # if multiple users are returned, just grab the first one Â¯\_(ãƒ„)_/Â¯

@client.event
async def on_message(msg: discord.Message):
    if msg.content.strip() == "!dumble":
        # get discord user from message
        author = msg.author
        print(author)

        user = getUserData(str(author))
        if user == None:
            await msg.channel.send(
                "Oh no! "
                + author.display_name
                + " isn't signed up for Dumble. "
                + "Sign up here: https://airtable.com/shrVpwd24p1353Ukk"
            )
        else:
            await msg.channel.send(
                author.display_name
                + " has joined the Dumble matchmaking pool! @"
                + author.display_name
                + " Please check your DM's from this bot to proceed!"
            )
            await userInterface(author, msg.channel)
        return

    elif msg.content.startswith("!dumble"):
        await msg.channel.send(
            "Sorry, I didn't get that. I don't have a help command either, so "
            + "go pester Eric or Andrew to build one."
        )
        return

client.run(TOKEN)


import dill as pickle
from datetime import datetime
from constants import PREPOSITIONS
from collections import defaultdict
import re

caps_matcher = re.compile(r'[A-ZÄÖÜ]')
lemma_map = {}
lemmas = set()
lemma_file = open('lemmatization-de.txt')
for line in lemma_file.readlines():
    [lemma, token] = line.split()
    lemma_map[token] = lemma
    lemmas.add(lemma)

def pick_lemma_from_pair(matches):
    [a, b] = matches
    if a in lemma_map and lemma_map[a] == b:
        return b
    if b in lemma_map and lemma_map[b] == a:
        return a
    else:
        return min(matches, key=len)

# generalizes the above
# bad. bad. just returns the first match that is the lemma of another match
# probably will usually just return the shortest?
# could be more than one. no time to fix.
# ideally i'd use some kind of semantic analysis to sort candidates
def pick_lemma_from_many(matches):
    for match in matches:
        without_that_one = list(matches)
        without_that_one.remove(match)
        if match in lemma_map and lemma_map[match] in without_that_one:
            return lemma_map[match]
    shortest = min(matches, key=len)
    return shortest

def old_lemma_fn(yc):
    for fn, candidate in yc.fuge_candidates[0].items():
        if candidate == yc.old_lemmas[0]:
            return fn, True
    return 'NULL', False

t0 = datetime.now()
for year in range(1995, 2023):
    t1 = datetime.now()
    elapsed = t1 - t0
    t0 = t1
    print(f'{year}\t{elapsed}')
    ycs = pickle.load(open(f'dumps/named_ents_filtered/{year}ycs', 'rb'))
    countsfile = open(f'corpora/deu_news_{year}_1M/deu_news_{year}_1M-words.txt')
    counts = defaultdict(int)
    for line in countsfile.readlines():
        if len(line.split()) != 3:
            continue
        _, word, count = line.split()
        counts[word] = int(count)
    # messy af, out of time
    for text, yc in ycs.items():
        matches = []
        for fn, candidate in yc.fuge_candidates[0].items():
            if candidate in lemmas:
                matches.append(candidate)
        if len(matches) == 0:
            fn, found = old_lemma_fn(yc)
            if found:
                yc.lemma_found = True
                lemma = yc.fuge_candidates[0][fn]
            elif yc.splits[0].lower() not in PREPOSITIONS:
            #     dictionary lookup
                yc.lemma_found = False
                for fn, candidate in yc.fuge_candidates[0].items():
                    if candidate in lemma_map:
                        matches.append(candidate)
            else:
                yc.lemma_found = False
        elif len(matches) == 2:
            yc.lemma_found = True
            lemma = pick_lemma_from_pair(matches)
        elif len(matches) > 2:
            yc.lemma_found = True
            lemma = pick_lemma_from_many(matches)
        else:
            yc.lemma_found = True
            lemma = matches[0]

        if yc.lemma_found:
            candidates = yc.fuge_candidates[0].items()
            for fn, defuged in candidates:
                if lemma == defuged:
                    yc.chosen_fn = fn
                    break #?
        # pick best dictionary match
        elif len(matches) > 0:
            yc.chosen_fn = 'NULL'
            chosen_word = max(matches, key=lambda x: counts[x])
            candidates = yc.fuge_candidates[0].items()
            for fn, defuged in candidates:
                if chosen_word == defuged:
                    yc.chosen_fn = fn
                    if caps_matcher.match(yc.text[0]):
                        pass
                    break #?
        else:
            yc.chosen_fn = 'NULL'
    with open(f'dumps/lemma_lookup_trial/{year}ycs', 'wb') as out_file:
        pickle.dump(ycs, out_file)

# print(f'single matches: {single_match}')
# print(f'no matches: {no_match}')
# print(f'multi matches: {multi_match}')


from jax.api import jit, grad, eval_shape, jacobian, jvp, vjp

import jax.numpy as np

from jax import random

# from jax.experimental import optimizers

from jax.experimental.stax import logsoftmax

from functools import partial

import tempfile

import itertools as IT

import os

from jax.experimental import optimizers

optimizer_dict = {}
optimizer_dict['adam'] = optimizers.adam
optimizer_dict['sgd'] = optimizers.sgd

def accuracy(params, f, images, targets, key = None):
    target_class = np.argmax(targets, axis=1)

    if key is not None:
        predicted_class = np.argmax(f(params, images, rng = key), axis=1)
    else:
        predicted_class = np.argmax(f(params, images), axis=1)
    return np.mean(predicted_class == target_class)

# def mean_and_variation_acc_across_trials(results_mat):
#     """
#     Args:
#     results_mat: A `np.ndarray` of shape [n_trials, n_data_points_evaluated]

#     Returns:
#     av_results_mat_across_trials: averaged results across trials (axis 0).

#     max_min_diff_ntk_distance: the upper/lower limit results obtaiend in different trials centered acround the results of av_results_mat_across_trials

#     """
#     av_results_mat_across_trials =  np.float32(np.mean(results_mat, axis = 0))

#     max_min_in_trials = np.vstack( (np.amin(results_mat, axis = 0 ), np.amax(results_mat, axis = 0 ) ) )

#     max_min_centered_diff = np.abs(max_min_in_trials - av_results_mat_across_trials)

#     return av_results_mat_across_trials, max_min_centered_diff

def _read_keys(keys):
    if keys is None or (isinstance(keys, np.ndarray) and keys.shape == (2,)):
        key1 = key2 = keys
    elif isinstance(keys, tuple):
        # assuming x1 and x2 using key1 and key2, resp.
        key1, key2 = keys
    elif isinstance(keys, np.ndarray) and keys.shape == (2, 2):
        key1, key2 = keys[0], keys[1]
    else:
        raise ValueError('`keys` must be one of the following: `None`, a PRNG '
                     'key, a tuple of PRNG keys or a (2, 2) array and dtype '
                     'unint32')
    return key1, key2

def empirical_ntk_fn(f):
    """Computes the ntk without batching for inputs x1 and x2.
    The Neural Tangent Kernel is defined as J(X_1)^T J(X_2) where J is the
    jacobian df/dparams. Computing the NTK directly involves directly
    instantiating the jacobian which takes
    O(dataset_size * output_dim * parameters) memory. It turns out it is
    substantially more efficient (especially as the number of parameters grows)
    to compute the NTK implicitly.
    This involves using JAX's autograd to compute derivatives of linear functions
    (which do not depend on the inputs). Thus, we find it more efficient to refer
    to fx_dummy for the outputs of the network. fx_dummy has the same shape as
    the output of the network on a single piece of input data.
    TODO(schsam): Write up a better description of the implicit method.
    Args:
    f: The function whose NTK we are computing. f should have the signature
       f(params, inputs) and should return an `np.ndarray` of outputs with shape
       [|inputs|, output_dim].
    Returns:
    A function ntk_fn that computes the empirical ntk.
    """

    def ntk_fn(x1, x2, params, keys=None):
        """Computes the empirical ntk.
        Args:
          x1: A first `np.ndarray` of inputs, of shape [n1, ...], over which we
            would like to compute the NTK.
          x2: A second `np.ndarray` of inputs, of shape [n2, ...], over which we
            would like to compute the NTK.
          params: A PyTree of parameters about which we would like to compute the
            neural tangent kernel.
          keys: None or a PRNG key or a tuple of PRNG keys or a (2, 2) array and
            dtype uint32. If `key == None`, then the function `f` is deterministic
            and requires no PRNG key; else if `keys` is a single PRNG key, then x1
            and x2 must be the same and share the same PRNG key; else x1 and x2 use
            two different PRNG keys.
        Returns:
          A `np.ndarray` of shape [n1, n2] + output_shape + output_shape.
        """
        key1, key2 = _read_keys(keys)
        # TODO(xlc): find a good way to check utils.x1_is_x2(x1, x2) == (key1==key2)
        if x2 is None:
            x2 = x1

        f_dummy = partial(f, rng=random.PRNGKey(1))
        fx2_struct = eval_shape(f_dummy, params, x2)
        fx_dummy = np.ones(fx2_struct.shape, fx2_struct.dtype)
        def delta_vjp_jvp(delta):
            def delta_vjp(delta):
                return vjp(lambda p: f(p, x2, rng=key2), params)[1](delta)
            return jvp(lambda p: f(p, x1, rng=key1), (params,), delta_vjp(delta))[1]

        ntk = jacobian(delta_vjp_jvp)(fx_dummy)
        ndim = len(fx2_struct.shape)
        ordering = (0, ndim) + tuple(range(1, ndim)) + \
            tuple(x + ndim for x in range(1, ndim))
        return np.transpose(ntk, ordering)

    return ntk_fn

def get_masks_from_jax_params(params, nn_density_level, magnitude_base_bool = True, global_bool = False, reshuffle_seed = 0):
    """ Assemble a collection of 0-1 valued masks which are of the same sizes and shapes as layers' weight tensors
        Note that this function ignores bias parameters.

    Args:
        params: parameters in a jax.experimental.stax format.
        nn_density_level: the desired density level for weight parameters.
        magnitude_base_bool: a boolean variable that decides whether to prune the network by magnitude or randomly prune the network

    Returns:
        masks: a collection of 0-1 valued masks which are of the same sizes and shapes as the layers' weight tensors.
    """

    if (type(magnitude_base_bool) != bool) or (type(global_bool) != bool):
        raise ValueError("magnitude_base_bool and global_bool should be boolean variables")

    masks = []

    if global_bool:
        weight_magnitudes_pooled = np.concatenate([ np.abs(layer_params[0].flatten()) for layer_params in params if len(layer_params) > 1])
        idx = int( (1 - nn_density_level) * np.size(weight_magnitudes_pooled) )
        global_thres = np.sort(weight_magnitudes_pooled)[idx]

    for layer_index in range( len(params)):

        if len(params[layer_index]) < 2:
            # In this the case, the layer does not contain weight and bias parameters.
            masks.append( [] )

        elif len(params[layer_index]) == 2:
            # In this case, the layer contains a tuple of parameters for weights and biases

            weights = params[layer_index][0]

            weight_magnitudes = np.abs(weights)

            if global_bool and magnitude_base_bool:

                this_mask = np.float32(weight_magnitudes > global_thres)

            else:
                # index: number of pruned parameters
                idx = int( (1 - nn_density_level) * np.size(weights) )

                # threshold: entries which below the thredhold will be removed
                thres = np.sort(np.reshape(weight_magnitudes, [-1] ))[idx]

                # 0 selected for weight parameters with magnitudes smaller than the threshold, 1 otherwise
                this_mask = np.float32(weight_magnitudes > thres)

                if magnitude_base_bool == False:
                    # in the case of random pruning: randomly shuffle the mask
                    this_mask = random.shuffle(random.PRNGKey(0), this_mask )

            masks.append(this_mask )

        else:
            raise NotImplementedError

    return masks

def get_sparse_params_filtered_by_masks(params, masks):

    """ Filter jax.experimental.stax parameters according to a collection of masks

    Args:
        params: parameters in a stax format.
        masks: a collection of masks for parameter pruning.

    Returns:
        sparse_params: sparse parameters acquired by passing params through the binay masks.
    """

    sparse_params = []
    for layer_index in range( len(params)):

        if len(params[layer_index]) < 2:
            # In this the case, the layer does not contain weight or bias parameters
            sparse_params.append( () )

        elif len(params[layer_index]) == 2:
            # In this case, the layer contains a tuple of parameters for weights and biases
            weights  = params[layer_index][0]

            biases = params[layer_index][1]

            mask_this_layer = masks[layer_index]

            # sparse weights gated by masks
            sparse_weights = np.multiply(mask_this_layer, weights )

            # weight-bias tuple
            sparse_params_this_layer = (sparse_weights, biases )

            sparse_params.append( sparse_params_this_layer )
        else:
            raise NotImplementedError

    return sparse_params

def stax_params_l2_square(params, mask = None, regularize_bias_bool = False):

    """ Compute squared l2 norm of stax parameters

    Args:
        params: parameters in a stax format.
        regularize_bias_bool: a boolean var deciding wether biases are regularized or not.
    Returns:
        params_norm_squared: squared l2 norm of stax parameters.
    """
    if (type(regularize_bias_bool) != bool):
        raise ValueError("regularize_bias_bool should be a boolean variable")

    if regularize_bias_bool == False:
        # assemble a list of weight parmeters; discard bias parameters
        list_params = [layer_param[0] for layer_param in params if len(layer_param) == 2 ]
    else:
        # remove the empty tuple from the list of paramters.
        list_params = list(sum(params, ()))

    params_norm_squared = np.sum([ np.sum(np.square( list_params[i] ))  for i in range(len(list_params)) ]  )

    return params_norm_squared

def softmax_cross_entropy_with_logits_l2_reg(params, f, x, targets, masks = None, L2_REG_COEFF = 0.0, key = None):

    """ cross entropy loss + weighted l2 regularization.

    Args:
        params: parameters in a stax format.
        apply_fn: a function that maps a set of network-parameters together with a set of network-inputs to network-outputs.
        x: network inputs
        targets: the target outputs
        masks: the sparsity-inducing mask
        L2_REG_COEFF: l2 regularization constant.
    Returns:
        The cross entropy loss + weighted l2 regularization
    """

    if masks is not None:
        masked_params = get_sparse_params_filtered_by_masks(params, masks)
    else:
        masked_params = params

    if key is not None:
        dense_outputs = f(masked_params, x, rng = key)
    else:
        dense_outputs = f(masked_params, x)

    preds = logsoftmax(dense_outputs)

    params_norm_squared = stax_params_l2_square(masked_params)

    return -np.mean(np.sum(preds * targets, axis=1)) + L2_REG_COEFF * params_norm_squared

def simpleaxis(ax):
    ax.spines['top'].set_visible(False)
    ax.spines['right'].set_visible(False)
    ax.get_xaxis().tick_bottom()
    ax.get_yaxis().tick_left()

# def get_snip_masks(params, nn_density_level, predict, snip_batch, batch_input_shape):

#     init_grads = grad(softmax_cross_entropy_with_logits_l2_reg)(params, predict, snip_batch[0].reshape(batch_input_shape), snip_batch[1])

#     cs = [abs( init_grads[idx][0] *  params[idx][0]).flatten() for idx in range(len(params)) if len(params[idx]) == 2 ]

#     pooled_cs = np.hstack(cs)

#     idx = int( (1 - nn_density_level) * len(pooled_cs) )

#     # threshold: entries which below the thredhold will be removed
#     thres = np.sort(pooled_cs)[idx]

#     masks = []
#     for layer_index in range( len(params)):

#         if len(params[layer_index]) < 2:
#             # In this the case, the layer does not contain weight and bias parameters.
#             masks.append( [] )

#         elif len(params[layer_index]) == 2:
#             # In this case, the layer contains a tuple of parameters for weights and biases

#             weights = params[layer_index][0]

#             weights_grad = init_grads[layer_index][0]

#             layer_cs = np.abs(weights * weights_grad)

#             # 0 selected for weight parameters with magnitudes smaller than the threshold, 1 otherwise
#             this_mask = np.float32(layer_cs >= thres)

#             masks.append(this_mask )

#         else:
#             raise NotImplementedError

#     return masks

def get_snip_masks(params, nn_density_level, predict, snip_batch, batch_input_shape, GlOBAL_PRUNE_BOOL = True):

    init_grads = grad(softmax_cross_entropy_with_logits_l2_reg)(params, predict, snip_batch[0].reshape(batch_input_shape), snip_batch[1])

    thres_list = [None] * len(params)

    if GlOBAL_PRUNE_BOOL == True: # global pruning

        cs = [abs( init_grads[idx][0] *  params[idx][0]).flatten() for idx in range(len(params)) if len(params[idx]) == 2 ]

        pooled_cs = np.hstack(cs)

        idx = int( (1 - nn_density_level) * len(pooled_cs) )

        # threshold: entries which below the thredhold will be removed
        thres = np.sort(pooled_cs)[idx]
        thres_list = [thres] * len(params)

    else: # layerwise pruning
        for layer_index in range( len(params)):
            if len(params[layer_index]) == 2:

                cs = abs( init_grads[layer_index][0] *  params[layer_index][0]).flatten()
                idx = int( (1 - nn_density_level) * len(cs) )
                # threshold: entries which below the thredhold will be removed
                thres = np.sort(cs)[idx]
                thres_list[layer_index] = thres

    masks = []
    for layer_index in range( len(params)):

        if len(params[layer_index]) < 2:
            # In this the case, the layer does not contain weight and bias parameters.
            masks.append( [] )

        elif len(params[layer_index]) == 2:
            # In this case, the layer contains a tuple of parameters for weights and biases

            weights = params[layer_index][0]

            weights_grad = init_grads[layer_index][0]

            layer_cs = np.abs(weights * weights_grad)

            # 0 selected for weight parameters with magnitudes smaller than the threshold, 1 otherwise
            this_mask = np.float32(layer_cs >= thres_list[layer_index])

            masks.append(this_mask )

        else:
            raise NotImplementedError

    return masks

def get_logit_snip_masks(params, nn_density_level, predict, x_batch, batch_input_shape, GlOBAL_PRUNE_BOOL = True):

    def norm_square_logits(params, f, x):
        return np.sum(f(params, x) **2)

    init_grads = grad(norm_square_logits)(params, predict, x_batch.reshape(batch_input_shape) )

    thres_list = [None] * len(params)

    if GlOBAL_PRUNE_BOOL == True: # global pruning

        cs = [abs( init_grads[idx][0] *  params[idx][0]).flatten() for idx in range(len(params)) if len(params[idx]) == 2 ]

        pooled_cs = np.hstack(cs)

        idx = int( (1 - nn_density_level) * len(pooled_cs) )

        # threshold: entries which below the thredhold will be removed
        thres = np.sort(pooled_cs)[idx]
        thres_list = [thres] * len(params)

    else: # layerwise pruning
        for layer_index in range( len(params)):
            if len(params[layer_index]) == 2:

                cs = abs( init_grads[layer_index][0] *  params[layer_index][0]).flatten()
                idx = int( (1 - nn_density_level) * len(cs) )
                # threshold: entries which below the thredhold will be removed
                thres = np.sort(cs)[idx]
                thres_list[layer_index] = thres

    masks = []
    for layer_index in range( len(params)):

        if len(params[layer_index]) < 2:
            # In this the case, the layer does not contain weight and bias parameters.
            masks.append( [] )

        elif len(params[layer_index]) == 2:
            # In this case, the layer contains a tuple of parameters for weights and biases

            weights = params[layer_index][0]

            weights_grad = init_grads[layer_index][0]

            layer_cs = np.abs(weights * weights_grad)

            # 0 selected for weight parameters with magnitudes smaller than the threshold, 1 otherwise
            this_mask = np.float32(layer_cs >= thres_list[layer_index])

            masks.append(this_mask )

        else:
            raise NotImplementedError

    return masks

def mean_and_var_across_trials(results_mat):
    """
    Args:
    results_mat: A `np.ndarray` of shape [n_trials, n_data_points_evaluated]

    Returns:
    av_results_mat_across_trials: averaged results across trials (axis 0).

    var_results_mat_across_trials: the variance across trials
    """

    av_results_mat_across_trials =  np.float32(np.mean(results_mat, axis = 0))
    var_results_mat_across_trials = np.var(results_mat, axis = 0)
#     max_min_in_trials = np.vstack( (np.amin(results_mat, axis = 0 ), np.amax(results_mat, axis = 0 ) ) )

#     max_min_centered_diff = np.abs(max_min_in_trials - av_results_mat_across_trials)

    return av_results_mat_across_trials, var_results_mat_across_trials

def load_supervised_training_result(supervised_result_file, wiring_str, nn_density_list, num_plots = -1, supervised_result_path = '/tungstenfs/scratch/gzenke/liutian/nt_transfer/saved_data/supervised_results/'):
    result_dict = {}
    final_result_dict = {}

#     if wiring_str == 'rand':
        # layerwise and global random pruning is the same thing because each parameter will be pruned with an identitical chance.
#         supervised_result_file=supervised_result_file.replace("global", "layerwise")

    for density_level in nn_density_list:

        model_path = supervised_result_path + supervised_result_file
        model_wiring_dir = supervised_result_file +  '_' + wiring_str

        loadFile =  model_path + '/' + model_wiring_dir + '/density_' + str(density_level) + '/supervised_trained_' + supervised_result_file + '_' + 'density_' + str(density_level)

        density_result = list(np.load(loadFile + '.npy' ,allow_pickle='TRUE'))

        density_str = str(density_result[0])

        av_train, trial_variation_train = mean_and_var_across_trials(density_result[1][:, :num_plots] )

        av_test, trial_variation_test = mean_and_var_across_trials(density_result[2][:, :num_plots])

#         av_train, trial_variation_train = mean_and_var_across_trials(density_result[1])

#         av_test, trial_variation_test = mean_and_var_across_trials(density_result[2])

        result_density_dict = {'train': {'av': av_train, 'var': trial_variation_train}, 'test': {'av': av_test, 'var': trial_variation_test}}

#         final_result_density_dict = {'train': {'av': av_train[-1], 'var': trial_variation_train[:, -1]}, 'test': {'av': av_test[-1], 'var': trial_variation_test[:, -1]}}
        final_result_density_dict = {'train': {'av': av_train[-1], 'var': trial_variation_train[ -1]}, 'test': {'av': av_test[-1], 'var': trial_variation_test[ -1]}}

        result_dict[str(density_level) ] = result_density_dict

        final_result_dict[str(density_level) ] = final_result_density_dict

    return result_dict, final_result_dict


__author__ = 'Marvin Smith'

#  Python Libraries
import random, math, logging

class Exit_Condition(object):

    max_repeats      = None
    repeated_counter = 0
    current_best     = None

    def __init__(self, max_repeats = 100):

        #  Set the max repeat value
        self.max_repeats = max_repeats

    def Check_Exit(self, test_fitness):
        """
        Evaluate the candidate item to see if we have reached the desired exit condition
        :param candidate:
        :return:
        """

        #  Check if we have even started yet
        if self.current_best is None:
            self.current_best     = test_fitness
            self.repeated_counter = 0
            return False

        #  If the candidate and best are equivalent
        elif math.fabs(test_fitness - self.current_best) < 1:
            self.repeated_counter+= 1
            if self.repeated_counter >= self.max_repeats:
                return True
            else:
                return False

        #  Otherwise, they are not the same
        else:
            self.repeated_counter = 0
            if self.current_best < test_fitness:
                logging.debug("warning: replacing the current exit test with an inferior result")
            self.current_best     = test_fitness
            return False

class Genetic_Algorithm(object):

    #  Current Population
    population = None

    #  Fitness Function
    fitness_function = None

    #  Selection Rate
    selection_rate = None

    #  Crossover Algorithm
    crossover_algorithm = None

    #  Mutation Algorithm
    mutation_algorithm = None

    #  Mutation Rate
    mutation_rate = None

    #  Preservation rate
    preservation_rate = None

    #  Exit Object
    exit_object = None

    #----------------------------#
    #-        Constructor       -#
    #----------------------------#
    def __init__(self, population,
                       fitness_function,
                       crossover_algorithm,
                       mutation_algorithm,
                       selection_rate=0.25,
                       mutation_rate=0.02,
                       preservation_rate=0.01,
                       exit_on_repeats=1000):

        self.exit_object = Exit_Condition(exit_on_repeats)

        #  Set the population
        self.population = population

        #  Set the fitness function
        self.fitness_function = fitness_function

        #  Set the selection_rate
        self.selection_rate = selection_rate

        #  Set the crossover algorithm
        self.crossover_algorithm = crossover_algorithm

        #  Set the mutation algorithm
        self.mutation_algorithm = mutation_algorithm

        #  Set the mutation rate
        self.mutation_rate = mutation_rate

        #  Set the preservation rate
        self.preservation_rate = preservation_rate

    def Run_Iterations(self, max_iterations):

        #  Size of the subset
        subset_size = int(self.selection_rate * len(self.population))
        preserve_size = int(self.preservation_rate * len(self.population))
        mutation_size = int(self.mutation_rate * len(self.population))

        #  Population size
        pop_size = len(self.population)

        #  For each iteration
        for i in range(0, max_iterations):

            #  Select Preservation Set
            preservation_list = self.population[0:preserve_size]

            #  Run Selection
            sub_population = self.population[0:subset_size]

            #  Create result population
            self.population = preservation_list + sub_population

            #  Run Crossover
            while len(self.population) < pop_size:

                #  Select two indeces
                idA = random.randint(0, subset_size-1)
                idB = random.randint(0, subset_size-1)

                #  Merge two samples
                samples = self.crossover_algorithm(sub_population[idA],
                                                   sub_population[idB])

                #  Add the samples
                if len(sub_population) < pop_size:
                    self.population.append(samples[0])
                if len(sub_population) < pop_size:
                    self.population.append(samples[1])

            #  Run Mutation
            for x in range(preserve_size,mutation_size):

                #  Select a random index
                idx = random.randint(preserve_size, len(self.population)-1)
                self.mutation_algorithm(self.population[idx])

            #  Recompute fitness
            for x in range(0,len(self.population)):
                self.population[x].Update_Fitness()

            #  Sort
            self.population = sorted(self.population,
                                     key=self.fitness_function)

            #  Exit Condition
            if self.exit_object.Check_Exit(self.population[0].distance) == True:
                logging.debug('Exiting early on exit condition.')
                break

        #  Return the population
        return self.population


# Naive Bayes Classification
# Made by: Khalid Qubbaj, Taras Derewecki, Raj Zaveri

# Import Libraries
import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.model_selection import cross_val_score

# Load Data Set
df2 = pd.read_csv("train.csv", index_col="RecipeId")
df2.target = df2.Target.astype('category')

# Load Training Set
dfTrain = pd.read_csv("train.csv", index_col="RecipeId")
trainY = dfTrain.Target.astype('category')
dfTrain = dfTrain.drop(["Target"],axis=1)
df = pd.concat([dfTrain, pd.read_csv("test.csv", index_col="RecipeId")])

# Stemming - Imports punkt
import nltk
from nltk.stem.porter import PorterStemmer
nltk.download('punkt')

def tokenize(text):
    tokens = nltk.word_tokenize(text)
    stems = []
    for item in tokens:
        stems.append(PorterStemmer().stem(item))
    return stems

# Fit Classifier the training data
v = TfidfVectorizer(max_features=1800, tokenizer=tokenize)
df_x = v.fit_transform(dfTrain.Ingredients.values.astype('U')).todense()

#print(v.get_feature_names())

            # INDIAN
# ndx=v.get_feature_names().index("indian")
# df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index("ghee")
df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index("chutney")
# df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index('masala')
# df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index('turmer')
# df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index("masa")
df_x[:,ndx] = df_x[:,ndx] * 20

            # FRENCH
ndx=v.get_feature_names().index('french')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index('baguett')
df_x[:,ndx] = df_x[:,ndx] * 20

            # CHINESE
ndx=v.get_feature_names().index('chines')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index("szechwan")
df_x[:,ndx] = df_x[:,ndx] * 20

            # GREEK
ndx=v.get_feature_names().index('greek')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index('feta')
df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index("yoghurt")
# df_x[:,ndx] = df_x[:,ndx] * 20

            # ITALIAN
ndx=v.get_feature_names().index('italian')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index("bertolli")
df_x[:,ndx] = df_x[:,ndx] * 20

            # MEXICAN
ndx=v.get_feature_names().index('mexican')
df_x[:,ndx] = df_x[:,ndx] * 20
ndx=v.get_feature_names().index('salsa')
df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index("burrito")
# df_x[:,ndx] = df_x[:,ndx] * 20
#ndx=v.get_feature_names().index("tequila")
#df_x[:,ndx] = df_x[:,ndx] * 20

            # THAI
ndx=v.get_feature_names().index('thai')
df_x[:,ndx] = df_x[:,ndx] * 20

    # OTHERS THAT SEEM TO IMPROVE ACC
# ndx=v.get_feature_names().index("epazot")
# df_x[:,ndx] = df_x[:,ndx] * 20
# ndx=v.get_feature_names().index("haricot")
# df_x[:,ndx] = df_x[:,ndx] * 20

# Import Naive Bayes
from sklearn.naive_bayes import MultinomialNB

# Get the best alpha value [0.5, 1, 10]
bestAlpha = None
#for alpha in [0.01, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1, 10]:
for alpha in [0.3]:
    nb = MultinomialNB(alpha=alpha, fit_prior=True, class_prior=None)
    scores = cross_val_score(nb, df_x, trainY, cv=10)

    print("For alpha =", alpha, " mean accuracy accross folds =", scores.mean(), " standard deviation across folds =",
          scores.std())
    if bestAlpha == None or scores.mean() > bestAlphaMeanAcc:
        bestAlpha = alpha
        bestAlphaMeanAcc = scores.mean()

print("Best k =", bestAlpha, "with estimated accuracy of", bestAlphaMeanAcc)

# Fit data into Naive Bayes
nb = MultinomialNB(alpha=bestAlpha, fit_prior=True, class_prior=None)
df_x = v.fit_transform(df.Ingredients.values.astype('U')).todense()
trainX = df_x[:len(dfTrain)]
testX = df_x[len(dfTrain):]
nb.fit(trainX,trainY)

# Generate Submission file
predictions = nb.predict(testX)
submissions = pd.read_csv("sampleSubmission.csv", index_col="RecipeId")
submissions.Target = predictions
submissions.to_csv("submission.csv")


import os
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import torch
from matplotlib.gridspec import GridSpec
import matplotlib.patches as mpatches
from os.path import join
import random
# from trial.plot_scripts.secondary_results import per_user_results

from trial.plot_scripts.util import collect_all_user_data, get_usefull_data_from_df

def subplot_overview_correct_false(data, ax=None):
    if ax is None:
        ax = plt.gca()

    num_correct = sum(data.Correct)
    num_total = len(data.Correct)
    num_wrong = num_total - num_correct
    labels = 'Correct', 'Wrong'
    plotdata = [num_correct, num_wrong]
    subplt = ax.pie(plotdata, explode=(0.1, 0.1), shadow=True, startangle=80, labels=labels, autopct='%1.1f%%',textprops={'fontsize': 14, 'fontweight':'bold'}, colors=('C0', 'red'), wedgeprops={"edgecolor":"k",'linewidth': 2, 'antialiased': True})
    ax.set_title("Top 1")

    for autotext in subplt[2]:
        autotext.set_color('white')
    return subplt

def subplot_confusion_matrix(data, ax=None):
    class ConfusionMatrix(object):
        def __init__(self, n_classes, labels=None):
            self.n_classes = n_classes
            self.mat = torch.zeros(n_classes, n_classes, requires_grad=False)
            self.labels = labels

        def update(self, preds, labels):
            for p, t in zip(preds, labels):
                self.mat[t, p] += 1

    predictions, gtlabels, labels, n_classes = get_usefull_data_from_df(data)
    label_to_idx = {l: idx for idx, l in enumerate(labels)}
    confusionmatrix = ConfusionMatrix(n_classes, labels)

    if ax is None:
        ax = plt.gca()

    for p,g in zip(data.Prediction, data.GroundTruth):
        confusionmatrix.update([label_to_idx[p]], [label_to_idx[g]])

    fontsize=15
    label_angle=45
    mat = confusionmatrix.mat

    total_samples = mat.sum(axis=1)
    mat = mat / total_samples[:, None]
    mat = torch.nan_to_num(mat, nan=0)

    subplt = ax.matshow(mat, cmap=plt.cm.Blues)

    threshold = mat.max() / 2.
    for (i, j), z in np.ndenumerate(mat):
        color = "white" if z > threshold else "black"
        ax.text(j, i, '{:.2f}'.format(z), ha='center', va='center', color=color, fontsize=fontsize)

    ax.set_xticks(range(0, n_classes))
    ax.xaxis.set_ticks_position('bottom')
    ax.set_xticklabels(labels, rotation=label_angle, fontsize=fontsize)
    ax.set_yticks(range(0, n_classes))
    ax.set_yticklabels(labels, rotation=label_angle, fontsize=fontsize)

    ax.set_ylabel('True Class')
    ax.yaxis.set_label_position("right")
    ax.set_xlabel('Predicted Class')
    ax.set_title("Confusion Matrix")
    return subplt

def subplot_confusion_histogram(data, ax=None):
    predictions, gtlabels, labels, n_classes = get_usefull_data_from_df(data)

    if ax is None:
        ax = plt.gca()

    num_correct_per_class = []
    num_wrong_per_class = []

    for lbl in labels:
        d = data.loc[data.GroundTruth==lbl]
        num_correct_per_class.append(len(d.loc[d.GroundTruth==d.Prediction]))
        num_wrong_per_class.append(len(d.loc[d.GroundTruth!=d.Prediction]))

    width = 0.5       # the width of the bars: can also be len(x) sequence

    subplt = ax.bar(labels, num_correct_per_class, width, label='Correct', edgecolor='black')
    ax.bar(labels, num_wrong_per_class, width, bottom=num_correct_per_class, label='Wrong', color='red', edgecolor='black')
    ax.set_xticklabels(labels, rotation=45, fontsize=15)
    bottom, top = ax.get_ylim()  # return the current ylim
    ax.set_ylim([bottom, top+(top*0.1)])
    ax.set_ylabel('# Videos')
    ax.set_title('Class confusion Histogram')
    ax.legend()

    return subplt

def subplot_decision_duration_chronological(data, ax=None):
    duration = list(data.Duration)
    correct = list(data.Correct)

    colors = []
    for i in correct:
        c = 'C0' if i==1 else 'red'
        colors.append(c)

    if ax is None:
        ax = plt.gca()

    subplt = ax.scatter(range(len(duration)), duration, c=colors)
    ax.set_ylabel('Viewing Duration (seconds)')
    ax.set_xlabel('Video-number shown')
    ax.set_title('How long to decide')
    ax.legend()

    return subplt

def subplot_decision_duration_per_class(data, ax=None, ylimtop=None):
    predictions, gtlabels, labels, n_classes = get_usefull_data_from_df(data)

    if ax is None:
        ax = plt.gca()

    duration_per_class = {}
    for i, lbl in enumerate(labels):
        durations = list(data.loc[data.GroundTruth==lbl].Duration)
        duration_per_class[lbl] = durations

        tmp = list(data.loc[data.GroundTruth==lbl].Correct)
        colors = []
        for tt in tmp:
            c = 'C0' if tt==1 else 'red'
            colors.append(c)

        x = np.random.normal(i+1, 0.08, size=len(durations))
        ax.scatter(x, durations, alpha=0.15,  s=20, c=colors)
        # ax.plot(x, durations, 'bo')

    ax.boxplot(duration_per_class.values(), widths=0.3, vert=True, showfliers=False, boxprops=dict(linewidth=2, color='black'), whiskerprops=dict(linestyle='-',linewidth=2.0, color='black'))
    ax.set_xticklabels(duration_per_class.keys(), fontsize=15)

    red_patch = mpatches.Patch(color='red', label='Wrong')
    blue_patch = mpatches.Patch(color='C0', label='Correct')

    ax.legend(handles=[blue_patch, red_patch], loc='upper right')
    ax.set_ylim([0, ylimtop])
    ax.set_ylabel('Viewing Duration (seconds)')
    # ax.set_xlabel('Video-number shown')
    ax.set_title('Decision speed')
    ax.set_xticklabels(labels, rotation=45)

    # ax.legend()

    return

def main_from_csv(filename):
    df_all = pd.read_csv(filename)
    main(df_all)

def main(dataframe, save_dir=None):

    df_all = dataframe.copy()
    df_all = df_all.loc[df_all.Trainmode==0]

    # TODO: SET THIS MAX YLIM TO SOMETHING REASONABLE
    max_decision_time = round(max(list(df_all.Duration))) + 2
    max_decision_time = 30
    modalities = set(df_all.Modality)

    modalities_nicename = {
        "ucf5": "UCF5",
        "afd5": "AFD5",
        # "ucf5rgbflow": "DONT USE RGBFLOW"
    }

    for modality in ['ucf5', 'afd5']:
        df = df_all.loc[df_all.Modality==modality]
        f, axes = plt.subplots(1, 4, figsize=(16, 5))
        (ax1, ax2, ax3, ax4) = axes.ravel()
        subplot_overview_correct_false(df, ax=ax1)
        subplot_confusion_matrix(df, ax=ax2)
        subplot_decision_duration_per_class(df, ax=ax3, ylimtop=max_decision_time)
        subplot_confusion_histogram(df, ax=ax4)

        f.suptitle(f"{modalities_nicename[set(df.Modality).pop()]}")
        plt.show(block=False)

        if save_dir != None:
            f.savefig(f'{join(save_dir, modality + ".pdf")}')
        f.tight_layout()
    plt.tight_layout()
    plt.show(block=True)

if __name__ == "__main__":
    root_dir = '/home/f/projects/MotionPaper/trial/output/'
    root_output_dir = '/home/f/projects/MotionPaper/trial/output'
    df = collect_all_user_data(root_dir)

    main(df, save_dir=root_dir)
    # per_user_results()


import sys
from collections import deque

n, m = map(int,sys.stdin.readline().split())
matrix = [list(map(int, sys.stdin.readline().split())) for _ in range(n)]
c = [[-1]*m for _ in range(n)]
dx = [-1,1,0,0]
dy = [0,0,-1,1]

def bfs(x, y):
    queue = deque()
    queue.append((x,y))

    while queue:
        cor = queue.popleft()

        for i in range(4):
            nx = cor[0] + dx[i]
            ny = cor[1] + dy[i]

            if 0 <= nx < n and 0 <= ny < m:
                if matrix[nx][ny] == 1 and c[nx][ny] == -1:
                    c[nx][ny] = c[cor[0]][cor[1]] + 1
                    queue.append((nx,ny))

for k in range(n):
    for h in range(m):
        if matrix[k][h] == 0:
            c[k][h] = 0

for i in range(n):
    for j in range(m):
        if matrix[i][j] == 2:
            c[i][j] = 0
            bfs(i,j)

for a in range(n):
    for b in range(m):
        print(c[a][b], end=" ")
    print()


from bs4 import BeautifulSoup
from .config import categories_config
from .config import product_config
from functools import partial

bs_parser = partial(BeautifulSoup, features='html.parser')

def parse_categories(html):
    """
    Given a html page containing a list of categories, produce a list of category names and urls
    :param html: the html of the page
    :return: a list of objects with url and title set
    """
    parsed = bs_parser(html)
    categories = []
    for li in parsed.find(id=categories_config['category_container_identifier']).findAll('li'):
        a = li.findAll('a')[0]
        href = a.get('href')
        title = a.string
        categories.append({'title': title, 'url': href})
    return categories

def parse_product_links_from_category_page(html):
    """
    Given a html page representing a category, extract the links to the products from it
    :param html: the html page
    :return: a list of url strings
    """
    links = []
    parsed = bs_parser(html)\
        .find(id=categories_config['category_page_products_container'])\
        .findAll(
            'div',
            {'class': categories_config['category_page_product_link_container']}
        )
    for link_container in parsed:
        for a in link_container.findAll('a'):
            links.append(a.get('href'))
    return links

def parse_product_details(html):
    """
    Given a html page representing product details, extract the indvidual details into a hash
    :param html: the product page
    :return: a hash of product details
    """
    parsed = bs_parser(html)\
        .find(id=product_config['product_details_container'])

    price_div = parsed.findAll('div', {'class': 'price'})[0]

    prices = extract_prices_from_product_details_div(price_div)
    name = parsed.find('h1', {'class': 'productname'}).string
    item_number = parsed.find('span', {'itemprop': 'productID'}).string

    return {
        'name': name,
        'discounted_price': format_price(prices['discounted_price']),
        'high_price': format_price(prices['high_price']),
        'item_number': item_number
    }

def extract_prices_from_product_details_div(price_div):
    """
    Prices appear in three formats: no discount, discounted and price range
    Try to parse all of them based on what markup is present
    :param price_div: the portion of html representing the price <div>
    :return: a hash with keys for discounted_price and high_price
    """
    prices = {}
    # check for a discounted price
    discounted_price = price_div.find('div', {'class': 'lowPrice'})
    if discounted_price is not None:
        prices['discounted_price'] = discounted_price.string
        prices['high_price'] = price_div.find('div', {'class': 'standardprice'}).string
        return prices
    # check for price range
    price_range_low_price = price_div.find('span', {'itemprop': 'lowPrice'})
    if price_range_low_price is not None:
        prices['discounted_price'] = price_range_low_price.string
        prices['high_price'] = price_div.find('span', {'itemprop': 'highPrice'}).string
        return prices
    prices['discounted_price'] = ''
    prices['high_price'] = price_div.find('div', {'class': 'salesprice'}).string
    return prices

def format_price(price):
    return price.strip('\n').strip('£')