#!/usr/bin/env python3
import sys
import logging
import argparse
import serial
import asyncio
import pylitejet

async def cmd_none(args):
    print("Nothing to do. See -h.")

# Command: list
async def cmd_list(lj, args):
    if args.loads:
        load_states = await lj.get_all_load_states()
        for number in lj.loads():
            level = await lj.get_load_level(number) if load_states[number] else 0
            if args.hide_off and level == 0:
                continue
            name = await lj.get_load_name(number)
            if level == 0:
                level_string = "off"
            elif level == 99:
                level_string = "on"
            else:
                level_string = "at " + str(level) + "%"
            print('Load {} is named "{}" and is {}'.format(number, name, level_string))
    if args.scenes:
        for number in lj.scenes():
            name = await lj.get_scene_name(number)
            print('Scene {} is named "{}"'.format(number, name))
    if args.buttons:
        switch_states = await lj.get_all_switch_states()
        for number in lj.button_switches():
            if args.hide_off and not switch_states[number]:
                continue
            name = await lj.get_switch_name(number)
            is_pressed = " and is pressed" if switch_states[number] else ""
            print('Switch {} is named "{}"{}'.format(number, name, is_pressed))
    if args.all_switches:
        switch_states = await lj.get_all_switch_states()
        for number in lj.all_switches():
            if args.hide_off and not switch_states[number]:
                continue
            name = await lj.get_switch_name(number)
            is_pressed = " and is pressed" if switch_states[number] else ""
            print('(All) Switch {} is named "{}"{}'.format(number, name, is_pressed))

# Command: load
async def cmd_load(lj, args):
    name = await lj.get_load_name(args.number)
    level = await lj.get_load_level(args.number)
    if level == 0:
        level_string = "off"
    elif level == 99:
        level_string = "on"
    else:
        level_string = "at " + str(level) + "%"
    print(f'Load {args.number} is named "{name}" and is {level_string}')

async def cmd_load_on(lj, args):
    await lj.activate_load(args.number)
    await cmd_load(lj, args)

async def cmd_load_off(lj, args):
    await lj.deactivate_load(args.number)
    await cmd_load(lj, args)

async def cmd_load_set(lj, args):
    await lj.activate_load_at(args.number, args.level, args.rate)
    await cmd_load(lj, args)

# Command: switch
async def cmd_scene(lj, args):
    name = await lj.get_scene_name(args.number)
    print('Scene {} is named "{}"'.format(args.number, name))

async def cmd_scene_on(lj, args):
    await lj.activate_scene(args.number)
    await cmd_scene(lj, args)

async def cmd_scene_off(lj, args):
    await lj.deactivate_scene(args.number)
    await cmd_scene(lj, args)

# Command: switch
async def cmd_switch(lj, args):
    name = await lj.get_switch_name(args.number)
    print('Switch {} is named "{}"'.format(args.number, name))

async def cmd_switch_press(lj, args):
    await lj.press_switch(args.number)
    await cmd_switch(lj, args)

async def cmd_switch_release(lj, args):
    await lj.release_switch(args.number)
    await cmd_switch(lj, args)

# Command: monitor
async def cmd_monitor(lj: pylitejet.LiteJet, args):
    def capture(func, name, number):
        return lambda *args: func(name, number, *args)

    def load_activated(name, number, level):
        print("Load {} ({}) activated to {}%.".format(name, number, level or "??"))

    def load_deactivated(name, number, level):
        print("Load {} ({}) deactivated.".format(name, number))

    def switch_pressed(name, number):
        print("Switch {} ({}) pressed.".format(name, number))

    def switch_released(name, number):
        print("Switch {} ({}) released.".format(name, number))

    def connected_changed(connected: bool, reason: str):
        print("Connected" if connected else f"Disconnected {reason}")

    lj.on_connected_changed(connected_changed)

    for number in lj.loads():
        name = await lj.get_load_name(number)
        lj.on_load_activated(number, capture(load_activated, name, number))
        lj.on_load_deactivated(number, capture(load_deactivated, name, number))
    for number in lj.all_switches():
        name = await lj.get_switch_name(number)
        lj.on_switch_pressed(number, capture(switch_pressed, name, number))
        lj.on_switch_released(number, capture(switch_released, name, number))

    loop = asyncio.get_event_loop()
    await loop.run_in_executor(None, input, "Press any key to stop monitoring...")

# Main

async def main():
    parser = argparse.ArgumentParser("Control a LiteJet lighting system.")
    parser.set_defaults(func=None)
    parser.add_argument("--path", required=True)
    parser.add_argument(
        "-v",
        "--verbose",
        help="Show debug logging, including data sent and received via serial port.",
        action="store_const",
        const=logging.DEBUG,
        default=logging.WARN,
    )
    subparsers = parser.add_subparsers()

    parser_list = subparsers.add_parser("list", help="List available items.")
    parser_list.add_argument("-l", "--loads", action="store_true")
    parser_list.add_argument("-s", "--scenes", action="store_true")
    parser_list.add_argument("-b", "--buttons", action="store_true")
    parser_list.add_argument("--all_switches", action="store_true")
    parser_list.add_argument("--hide_off", action="store_true")
    parser_list.set_defaults(func=cmd_list)

    parser_load = subparsers.add_parser("load", help="Change load level.")
    parser_load.add_argument("number", type=int)
    subparser_load = parser_load.add_subparsers()
    parser_load_set = subparser_load.add_parser(
        "set", help="Set load to a specific level."
    )
    parser_load_set.add_argument("level", type=int)
    parser_load_set.add_argument("rate", type=int, default=0)
    parser_load_set.set_defaults(func=cmd_load_set)
    parser_load_on = subparser_load.add_parser(
        "on", help="Set load to its default level."
    )
    parser_load_on.set_defaults(func=cmd_load_on)
    parser_load_off = subparser_load.add_parser("off", help="Turn off a load.")
    parser_load_off.set_defaults(func=cmd_load_off)
    parser_load_get = subparser_load.add_parser("get", help="Get load information.")
    parser_load_get.set_defaults(func=cmd_load)

    parser_scene = subparsers.add_parser("scene", help="Change scene activation.")
    parser_scene.add_argument("number", type=int)
    subparser_scene = parser_scene.add_subparsers()
    parser_scene_on = subparser_scene.add_parser("on", help="Turn on a scene.")
    parser_scene_on.set_defaults(func=cmd_scene_on)
    parser_scene_off = subparser_scene.add_parser("off", help="Turn off a scene.")
    parser_scene_off.set_defaults(func=cmd_scene_off)
    parser_scene_get = subparser_scene.add_parser("get", help="Get scene information.")
    parser_scene_get.set_defaults(func=cmd_scene)

    parser_switch = subparsers.add_parser("switch", help="Change switch activation.")
    parser_switch.add_argument("number", type=int)
    subparser_switch = parser_switch.add_subparsers()
    parser_switch_on = subparser_switch.add_parser(
        "press", help="Simulate pressing a switch."
    )
    parser_switch_on.set_defaults(func=cmd_switch_press)
    parser_switch_off = subparser_switch.add_parser(
        "release", help="Simulate releasing a switch."
    )
    parser_switch_off.set_defaults(func=cmd_switch_release)
    parser_switch_get = subparser_switch.add_parser(
        "get", help="Get switch information."
    )
    parser_switch_get.set_defaults(func=cmd_switch)

    parser_monitor = subparsers.add_parser(
        "monitor", help="Monitor items for state changes."
    )
    parser_monitor.set_defaults(func=cmd_monitor)

    args = parser.parse_args()

    logging.basicConfig(level=args.verbose)

    serial.protocol_handler_packages.append("test_handlers")

    try:
        lj = await pylitejet.open(args.path)
    except pylitejet.LiteJetError as exc:
        print(f"Cannot connect: {exc}")
        return

    if args.func is not None:
        await args.func(lj, args)

    await lj.close()

asyncio.run(main())


import matplotlib.pyplot as plt
import pyfits

import matplotlib.patheffects

if 1:

    fits_name = "lambda_mollweide_halpha_fwhm06_0512.fits"
    f = pyfits.open(fits_name)
    data = f[1].data
    header = f[1].header

    # used fixed header
    del header["LONPOLE"]
    del header["LATPOLE"]

    fig = plt.figure(1, figsize=(8, 5))
    fig.clf()

    from pywcsgrid2.allsky_axes import make_allsky_axes_from_header

    ax = make_allsky_axes_from_header(fig, rect=111, header=header,
                                      lon_center=0.)

    if data is not None:
        im = ax.imshow(data, origin="lower", cmap=plt.cm.gray_r)
        im.set_clip_path(ax.patch)
        im.set_clim(0, 30)

    ef = matplotlib.patheffects.withStroke(foreground="w", linewidth=3)
    axis = ax.axis["lat=0"]
    axis.major_ticklabels.set_path_effects([ef])
    axis.label.set_path_effects([ef])
    axis.set_zorder(5)

    ax.grid()
    plt.show()


import numpy as np
import os
import pickle
from scipy.spatial.distance import cdist
import cv2
import matplotlib
matplotlib.use('agg')
import matplotlib.pyplot as plt
import matplotlib.patches as patches
import scipy.io as sio

def process_pool3_patch(lowpool_1, lowpool_2, row1, col1, row2, col2):
    pat1_core = lowpool_1[row1-1:row1+3,col1-1:col1+3].reshape(-1,256)
    pat2_core = lowpool_2[row2-1:row2+3,col2-1:col2+3].reshape(-1,256)
    cor_dist = cdist(pat1_core, pat2_core, 'cosine')

    sort_idx = np.argsort(cor_dist,axis=None)
    pat_dist_ls = []
    for ii in sort_idx[0:3]:
        pat1_idx_core, pat2_idx_core = np.unravel_index(ii, cor_dist.shape)
        pat1_r_core, pat1_c_core = np.unravel_index(pat1_idx_core, (4,4))
        pat2_r_core, pat2_c_core = np.unravel_index(pat2_idx_core, (4,4))
        pat1_r = pat1_r_core + (row1-1)
        pat1_c = pat1_c_core + (col1-1)
        pat2_r = pat2_r_core + (row2-1)
        pat2_c = pat2_c_core + (col2-1)

        rr1 = np.max((0,pat1_r-3))
        rr2 = np.max((0,pat2_r-3))
        cc1 = np.max((0,pat1_c-3))
        cc2 = np.max((0,pat2_c-3))

        # 6 by 6 context
        pat1 = lowpool_1[rr1:pat1_r+3, cc1:pat1_c+3]
        pat2 = lowpool_2[rr2:pat2_r+3, cc2:pat2_c+3]

        pat_dist = np.mean(cdist(pat1.reshape(-1,256), pat2.reshape(-1,256), 'cosine'),axis=None)
        pat_dist_ls.append(pat_dist)

    pat1_idx_core, pat2_idx_core = np.unravel_index(sort_idx[np.argmin(pat_dist_ls)], cor_dist.shape)
    pat1_r_core, pat1_c_core = np.unravel_index(pat1_idx_core, (4,4))
    pat2_r_core, pat2_c_core = np.unravel_index(pat2_idx_core, (4,4))

    pat1_r = pat1_r_core + (row1-1)
    pat1_c = pat1_c_core + (col1-1)
    pat2_r = pat2_r_core + (row2-1)
    pat2_c = pat2_c_core + (col2-1)

    return(pat1_r, pat1_c, pat2_r, pat2_c, 1-np.min(pat_dist_ls))

category = 'car'
set_type = 'train'
featDim = 256
offset = 2
dataset_root = '/home/yutong/dataset/'
list_dir = os.path.join(dataset_root, 'PASCAL3D+_release1.1','Image_sets')
proj_root ='/mnt/4TB_b/qing/VC_journal/'
cache_dir = os.path.join(proj_root, 'feat')

pool4_offset=2

root_dir = '/home/yutong/SPMatch/vp_examples_less/'
for vp in [0,45,90,135,180,225,270,315]:
# for vp in [45]:
    img_dir = os.path.join(root_dir, str(vp))
    # file list
    filelist = os.path.join(img_dir, 'file_list.txt')
    with open(filelist, 'r') as fh:
        contents = fh.readlines()

    img_list = [cc.strip() for cc in contents]
    img_idx2 = len(img_list)-1

    for img_idx1 in range(img_idx2):

        # load pool4 match result
        mat_filename = os.path.join(img_dir, 'img{}VSimg{}.mat'.format(img_idx1, img_idx2))
        mat_content = sio.loadmat(mat_filename)
        pool4_rst = mat_content['rst']

        filelist = os.path.join(img_dir, 'file_list.txt')
        with open(filelist, 'r') as fh:
            contents = fh.readlines()

        img_list = [cc.strip() for cc in contents]
        img_file1 = img_list[img_idx1]
        img_file2 = img_list[img_idx2]
        pool3_file1 = '.'.join(img_file1.split('.')[0:-1])+'_pool3.pickle'
        pool3_file2 = '.'.join(img_file2.split('.')[0:-1])+'_pool3.pickle'

        pool3_img1 = pickle.load(open(pool3_file1, 'rb'))
        pool3_img2 = pickle.load(open(pool3_file2, 'rb'))
        img1 = cv2.imread(img_file1)
        img2 = cv2.imread(img_file2)
        rratio = 224.0/np.min(img2.shape[0:2])
        img2 = cv2.resize(img2, (0,0), fx=rratio, fy=rratio)

        save_path = mat_filename.replace('.mat','_refine/')
        if not os.path.exists(save_path):
            os.makedirs(save_path)

        # to save pool4 and pool3 matched x1,y1,x1',y1'
        matched = [[],[]]

        for pi in range(pool4_rst.shape[0]):
            pool4_info = pool4_rst[pi]
            p4_score, img1_id, row1, col1, img2_id, row2, col2 = pool4_info.astype(int)

            pool3_row1 = (row1+pool4_offset-1)*2
            pool3_col1 = (col1+pool4_offset-1)*2
            pool3_row2 = (row2+pool4_offset-1)*2
            pool3_col2 = (col2+pool4_offset-1)*2

            # plot images
            plt.close()
            fig,ax = plt.subplots(1,2,figsize=(20,5))

            # Display the image
            ax[0].imshow(img1[:,:,::-1])
            ax[1].imshow(img2[:,:,::-1])

            pool4_matched = []
            # Create a Rectangle patch
            x_pool4 = 16*(col1+pool4_offset-1)-42
            y_pool4 = 16*(row1+pool4_offset-1)-42
            rect = patches.Rectangle((x_pool4,y_pool4),100,100,linewidth=1,edgecolor='r',facecolor='none')
            circ = patches.Circle((x_pool4+50,y_pool4+50),3)
            pool4_matched = pool4_matched + [x_pool4+50,y_pool4+50]
            ax[0].add_patch(rect)
            ax[0].add_patch(circ)

            x_pool4 = 16*(col2+pool4_offset-1)-42
            y_pool4 = 16*(row2+pool4_offset-1)-42
            rect = patches.Rectangle((x_pool4,y_pool4),100,100,linewidth=1,edgecolor='r',facecolor='none')
            circ = patches.Circle((x_pool4+50,y_pool4+50),3)
            pool4_matched = pool4_matched + [x_pool4+50,y_pool4+50]
            ax[1].add_patch(rect)
            ax[1].add_patch(circ)

            pool4_matched = pool4_matched + [p4_score]
            matched[0].append(pool4_matched)

            pool3_matched = []
            # refine the matches on pool3
            pat1_r, pat1_c, pat2_r, pat2_c, p3_score = process_pool3_patch(pool3_img1,pool3_img2, pool3_row1, pool3_col1, pool3_row2, pool3_col2)

            # Create a Rectangle patch -- pool3
            x_pool3 = 8*(pat1_c)-18
            y_pool3 = 8*(pat1_r)-18
            rect = patches.Rectangle((x_pool3,y_pool3),44,44,linewidth=1,edgecolor='r',facecolor='none')
            circ = patches.Circle((x_pool3+22,y_pool3+22),3, edgecolor='g',facecolor='g')
            pool3_matched = pool3_matched + [x_pool3+22,y_pool3+22]
            ax[0].add_patch(rect)
            ax[0].add_patch(circ)

            x_pool3 = 8*(pat2_c)-18
            y_pool3 = 8*(pat2_r)-18
            rect = patches.Rectangle((x_pool3,y_pool3),44,44,linewidth=1,edgecolor='r',facecolor='none')
            circ = patches.Circle((x_pool3+22,y_pool3+22),3, edgecolor='g',facecolor='g')
            pool3_matched = pool3_matched + [x_pool3+22,y_pool3+22]
            ax[1].add_patch(rect)
            ax[1].add_patch(circ)

            pool3_matched = pool3_matched + [p3_score]
            matched[1].append(pool3_matched)

            plt.savefig(os.path.join(save_path, '{}.png'.format(str(pi))))

        print(matched)
        save_pickle = mat_filename.replace('.mat','_matched.pickle')
        with open(save_pickle,'wb') as fh:
            pickle.dump(matched, fh)


import matplotlib.pyplot as plt
import numpy as np

from lib.core_simulation import (forestSimulation, forestConfig)
from lib.plots_from_simulation import (plots)

# 1. Prepare an initial state
# 2. Set a random cell on fire
# 3. Run the simulation until no cells are burning
# 4. Repeat 1-3 for n times and record the total number of cells burned

REPETITIONS = 10
INITIAL_DENSITIES = np.arange(0.28, 0.65, 0.02)

def prepare_initial_state():
    # initial state with one random burning cell
    initial_state = sim.random_world()
    initial_state[sim.rng.randint(0, config.height), sim.rng.randint(0, config.width)] = 2
    return initial_state

results = []

for density in INITIAL_DENSITIES:
  print(f"Running simulations for density {round(density, 3)}")
  config = forestConfig(width=100, height=100, p_tree=density,
                        p_sprout=0, p_propagate=0, p_lightning=0, seed=17)

  sim = forestSimulation(config)

  total_burned_results = []
  for i in range(REPETITIONS):
      sim.simulate(stop_when_no_burning=True, initial_state=prepare_initial_state())
      results += [[density, sum([frame[0] for frame in sim.stats]), len(sim.stats)]]

print(results)

fig = plt.figure()
ax1 = fig.add_subplot()
# plot the number of burned cells per density
burned_scatter = ax1.scatter([result[0] for result in results], [result[1] for result in results], label="number of burned cells")
ax1.set_ylabel("number of burned cells")
ax1.set_ylim(ymin=0)

# ax2 = ax1.twinx()
# # plot the number of steps per density
# steps_scatter = ax2.scatter([result[0] for result in results], [result[2]
#            for result in results], label="number of steps", color="gray")
# ax2.set_ylabel("number of steps")
# ax2.set_ylim(ymin=0)

plt.xlabel("initial density")
plt.legend(handles=[burned_scatter])
plt.show()


# Given an array of integers, calculate the ratios of its elements that are positive, negative, and zero. Print the decimal value of each fraction on a new line with  places after the decimal.

# Note: This challenge introduces precision problems. The test cases are scaled to six decimal places, though answers with absolute error of up to  are acceptable.

# Example

# There are  elements, two positive, two negative and one zero. Their ratios are ,  and . Results are printed as:

# 0.400000
# 0.400000
# 0.200000

# Function Description

# Complete the plusMinus function in the editor below.

# plusMinus has the following parameter(s):

# int arr[n]: an array of integers
# Print
# Print the ratios of positive, negative and zero values in the array. Each value should be printed on a separate line with  digits after the decimal. The function should not return a value.

# Input Format

# The first line contains an integer, , the size of the array.
# The second line contains  space-separated integers that describe .

# Constraints

# Output Format

# Print the following  lines, each to  decimals:

# 1. proportion of positive values
# 2. proportion of negative values
# 3. proportion of zeros

#!/bin/python3

import math
import os
import random
import re
import sys
from enum import Enum

class numberType(Enum):
    negative = "negative"
    positive = "positive"
    zero = "zero"

def plusMinus(arr):
    if arr is None or len(arr) == 0: return

    length = len(arr)
    counts = {}
    for item in arr:
        if item < 0:
            incrementCountOf(counts, numberType.negative)
        if item == 0:
            incrementCountOf(counts, numberType.zero)
        if item > 0:
            incrementCountOf(counts, numberType.positive)

    printRatioOf(counts, length, numberType.positive)
    printRatioOf(counts, length, numberType.negative)
    printRatioOf(counts, length, numberType.zero)

def incrementCountOf(counts:dict, of: numberType):
    if of in counts.keys():
        counts[of] += 1
    else:
        counts[of] = 1

def printRatioOf(counts: dict, length: int, of: numberType):
    print("%.6f"%(counts[of]/length))

if __name__ == '__main__':
    n = int(input().strip())

    arr = list(map(int, input().rstrip().split()))

    plusMinus(arr)


import text2emotion as te
from sklearn.feature_extraction.text import TfidfVectorizer

# Define the song lyric

def analyze_lyric(lyric):
    # Tokenize the lyric
    tokens = lyric.split()
    # Vectorize the tokens using TF-IDF
    vectorizer = TfidfVectorizer()
    vectorizer.fit_transform(tokens)
    vectorized_lyric = vectorizer.transform([lyric]).toarray()
    # Use the text2emotion library to predict the emotions in the lyric
    emotions = te.get_emotion(lyric)
    # Print the predicted emotions and the vectorized lyric
    # give me the most common emotion
    return emotions


from __future__ import division, print_function

import apache_beam as beam

class SelectFromNestedDict(beam.DoFn):
    """A :class:`~apache_beam.DoFn` for selecting subsets of records formed of
    nested dictionaries.

    Args:
        keys (list): list of dictionary keys to be selected. Each key is a
            string formed of dot separated words, each is used for selecting
            from a dict in the nested dicts. The order of the word in the
            "dot.separated.string" correspond to the dict level to select from.
            For instance, if the input record is ``{'a': {'b': 3, 'c': {'d': 2} }}``
            and keys is ``['a.b', 'a.c.d']``, the transform output will
            ``{'a_b': 3, 'a_c_d': 2} or {'b': 3, 'd': 2}`` depending on
            ``deepest_key_as_name `` below.
        deepest_key_as_name (bool): if set to True, the deepest selected fields
            keys will be used as names for the output dict keys.

    Examples:
        Select from records formed of nested dicts. ::

            import apache_beam as beam
            from apache_beam.options.pipeline_options import PipelineOptions
            from beam_nuggets.transforms import SelectFromNestedDict

            inputs = [
                {
                    'name': {'first': 'Star', 'second': 'Light'},
                    'address': {'st': 'Jupiter', 'flat': 3},
                    'email': 's@l.no'
                },
                {
                    'name': {'first': 'Mark', 'second': 'Sight'},
                    'address': {'st': 'Loon', 'flat': 5},
                    'email': 'm@s.no'
                }
            ]
            with beam.Pipeline(options=PipelineOptions()) as p:
                nested = p | "Reading nested dicts" >> beam.Create(inputs)
                transformed = nested | "filtering" >> beam.ParDo(SelectFromNestedDict(
                    keys=['name.first', 'address.st', 'email'],
                    # deepest_key_as_name=True,
                ))
                transformed | 'Writing to stdout' >> beam.Map(print)

        The output will be something like:

            {'address_st': 'Jupiter', 'name_first': 'Star', 'email': 's@l.no'}
            {'address_st': 'Loon', 'name_first': 'Mark', 'email': 'm@s.no'}

    """

    def __init__(self, keys, deepest_key_as_name=False, *args, **kwargs):
        super(SelectFromNestedDict, self).__init__(*args, **kwargs)
        self._compiled_keys = self._compile_keys(keys, deepest_key_as_name)

    def process(self, element):
        """
        Args:
            element(dict):
        """

        yield {
            out_key: self._retrieve(nested_keys, element)
            for nested_keys, out_key in self._compiled_keys
        }

    @staticmethod
    def _retrieve(nested_keys, element):
        for key in nested_keys:
            element = element[key]
        return element

    @staticmethod
    def _compile_keys(keys, deepest_key_as_name):
        def _get_out_dict_key(nested_keys):
            if deepest_key_as_name:
                return nested_keys[-1]
            else:
                return '_'.join(nested_keys)

        return [
            (
                nested_keys,  # ['a', 'b'] used for retrieving nested values
                _get_out_dict_key(nested_keys),  # 'a_b' or 'b'
            )
            for nested_keys in [key.split('.') for key in keys]  # ['a.b']
        ]


from enum import Enum
import boto3
import HTPredBenchCreator

class CELL(Enum):
    TSMC = 'tsmc_cells/'
    SYNOPSIS = 'synopsis_cells/'

def run(event, context):
    s3 = boto3.client('s3')
    file_name = "/tmp/afile"
    s3.download_file('htpred-bucket', event['s3key'], file_name)

    cell = CELL.TSMC.value if event['cell'] == 'tsmc' else CELL.SYNOPSIS.value

    outpf = '/tmp/file'

    if event['function'] == 'benchcreator':
        r = HTPredBenchCreator.Creator(file_name, cell, event['main_module'])
        s = open(outpf, 'w')
        s.write(r.convert())
        s.close()

    if event['function'] == 'featureextractor':
        r = HTPredBenchCreator.FeatureExtractor(file_name, cell, event['main_module'])
        r.getfeatures(outpf, ['preferences/mux_list.txt'])

    s3.upload_file(outpf, 'htpred-bucket', event['s3outputkey'])

    return {"STATUS":"SUCCESS"}


from django import forms
from django.conf import settings
from django.db.models import QuerySet
from snowpenguin.django.recaptcha3.fields import ReCaptchaField

from . import services
from .models import Product, Review

class ProductFilterForm(forms.Form):
    """Form for filtering products."""

    min_price = forms.DecimalField(
        min_value=0,
        decimal_places=2,
        label="",
        required=False,
        widget=forms.NumberInput(
            attrs={"class": "form-control mb-2", "placeholder": "Min price"}
        ),
    )
    max_price = forms.DecimalField(
        min_value=0,
        decimal_places=2,
        label="",
        required=False,
        widget=forms.NumberInput(
            attrs={"class": "form-control mb-2", "placeholder": "Max price"}
        ),
    )
    category = forms.ModelChoiceField(
        queryset=services.get_all_categories(),
        widget=forms.RadioSelect,
        required=False,
    )
    brands = forms.ModelMultipleChoiceField(
        queryset=services.get_all_brands(),
        widget=forms.CheckboxSelectMultiple,
        label="Brand",
        required=False,
    )
    years = forms.MultipleChoiceField(
        choices=[
            (year, year)
            for year in set(
                map(
                    lambda tup: int(tup[0]),
                    Product.objects.all().values_list("year"),
                )
            )
        ],
        widget=forms.CheckboxSelectMultiple,
        label="Year",
        required=False,
    )

    def get_filtered_(self, products: QuerySet[Product]) -> QuerySet[Product]:
        """Returns the filtered given products."""
        if self.cleaned_data.get("min_price"):
            products = products.filter(
                price__gte=self.cleaned_data.get("min_price")
            )
        if self.cleaned_data.get("max_price"):
            products = products.filter(
                price__lte=self.cleaned_data.get("max_price")
            )
        if self.cleaned_data.get("category"):
            products = products.filter(category=self.cleaned_data["category"])
        if self.cleaned_data.get("brands"):
            products = products.filter(brand__in=self.cleaned_data["brands"])
        if self.cleaned_data.get("years"):
            products = products.filter(year__in=self.cleaned_data["years"])
        return products

dict_with_field_classes = {"class": "form-control w-50 mb-2"}

class ReviewModelForm(forms.ModelForm):
    """Model form for adding review to product."""

    if not settings.TESTING:
        captcha = ReCaptchaField()

    class Meta:
        """Meta options for the ReviewForm."""

        model = Review
        fields = ("name", "body")
        if not settings.TESTING:
            fields = ("name", "body", "captcha")
        widgets = {
            "name": forms.TextInput(attrs=dict_with_field_classes),
            "body": forms.Textarea(attrs=dict_with_field_classes),
        }


import io

from unidecode import unidecode

DEBUG = False

# ======= Configuration =======
# Path to the table with the names of those present in the call
input_path = "../resources/attendance.csv" if DEBUG else "../resources/ed2905.csv"
# Path to the table with the names of those present in the call
output_path = "../resources/names.csv" if DEBUG else "../resources/names_table.csv"
# Minimum number of occurrences to register the presence
minimum_occurrence = 2 if DEBUG else 10

# Function: Generate a set with names from an attendence table
# Parameter: path = path of the table containing the data
# Parameter: minimum_occurrence = value of minimum number of occurrences to register the presence
def proccess(input_path, minimum_occurrence):
    text = io.open(input_path, "r+", encoding="utf8")
    cont = 0
    namedict = dict()
    for line in text:
        # Ignore the first two line of the table format
        if cont >= 2:
            line = line.rstrip()
            names = line.split(";")
            for name in names:
                if len(name) != 0:
                    namedict[name] = namedict.setdefault(name, 0) + 1
        cont += 1
    nameset = set()
    for name in namedict.keys():
        if namedict[name] >= minimum_occurrence:
            nameset.add(unidecode(name.title()))
    return nameset

# Function: Generate a set with names from an attendence table
# Parameter: output_path = path of csv with the names will be saved
# Parameter: namelist = set with the names to be write
def export(output_path, nameset):
    output_file = open(output_path, "w+", encoding="utf8")
    output_file.write("Nomes\n")
    namelist = list()
    for name in nameset:
        namelist.append(name)
    namelist.sort()
    for name in namelist:
        output_file.write(name)
        output_file.write("\n")

if __name__ == "__main__":
    names = proccess(input_path, minimum_occurrence)
    export(output_path, names)


# Standard Libs
import pytest
from unittest.mock import patch, MagicMock

# External Libs
from src.services.bank_transfer.service import BankTransferService
from tests.src.stubs.router_bank_accounts_stubs.stubs import x_thebes_bank_tuple
from decouple import Config

# stub
jwt_decompiled_stub = {
    "exp": 1678209788,
    "created_at": 1646673788.442139,
    "scope": {"view_type": "default", "user_level": "client"},
    "user": {
        "unique_id": "40db7fee-6d60-4d73-824f-1bf87edc4491",
        "nick_name": "RAST3",
        "portfolios": {
            "br": {"bovespa_account": "000000014-6", "bmf_account": "14"},
            "us": {"_": None},
        },
    },
}
jwt_data = {"CLIENT_JWT_NAME": "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9"}
bank_number = "001"
response_stub = {"agency": "0001", "bank": bank_number, "account": "000000014-6"}

from src.infrastructures.env_config import config

@pytest.mark.asyncio
@patch(
    "src.services.jwt.service.jwt_validator_and_decompile",
    return_value=jwt_decompiled_stub,
)
@patch(
    "src.services.jwt.service.verify_jwt_token_by_string",
    return_value=jwt_decompiled_stub,
)
@patch("decouple.Config.__call__", return_value=bank_number)
async def test_when_sending_the_right_jwt_then_return_the_bank_account_datas_from_jwt(
    mock_config_env_return,
    mock_jwt_validator_and_decompile,
    mock_verify_jwt_token_by_string,
):
    response = await BankTransferService.get_bank_transfer_account(
        request=MagicMock(headers=MagicMock(raw=x_thebes_bank_tuple))
    )
    assert response == response_stub


import os
import time
from google.oauth2.credentials import Credentials
from google_auth_oauthlib.flow import InstalledAppFlow
from googleapiclient.discovery import build
from googleapiclient.errors import HttpError

CLIENT_SECRETS_FILE = os.path.join(os.path.dirname(__file__), 'service_account.json')
SCOPES = ['https://www.googleapis.com/auth/youtube.force-ssl']
CHANNEL_ID = 'UC-lHJZR3Gqxm24_Vd_AJ5Yw'

def main():
    if os.path.exists('credentials.json'):
        credentials = Credentials.from_authorized_user_file('credentials.json', scopes = SCOPES)
    else:
        flow = InstalledAppFlow.from_client_secrets_file(CLIENT_SECRETS_FILE, scopes = SCOPES)
        credentials = flow.run_local_server(port=0)
        with open('credentials.json', 'w') as token:
            token.write(credentials.to_json())

    youtube = build('youtube', 'v3', credentials=credentials)
    nextPageToken = None

    while True:
        try:
            response = youtube.commentThreads().list(
                part = "id,snippet",
                allThreadsRelatedToChannelId = CHANNEL_ID,
                maxResults = 100,
                textFormat ='plainText',
                pageToken = nextPageToken
            ).execute()
            time.sleep(3)
        except HttpError as error:
            print(f'An error occurred: {error}')
            nextPageToken = None
            time.sleep(3)

        for item in response['items']:
            try:
                comment_id = item['snippet']['topLevelComment']['id']
                print ("comment ID: " + comment_id)
                youtube.comments().delete(id = comment_id).execute()
                time.sleep(3)
            except HttpError as error:
                print(f'An error occurred: {error}')
                time.sleep(3)

        nextPageToken = response.get('nextPageToken')
        if not nextPageToken:
            break

if __name__ == '__main__':
    main()


import requests
from requests import Session
from requests.adapters import HTTPAdapter
from urllib3 import Retry
from loguru import logger
from requests import HTTPError, Response

def get_session(add: dict, headers: dict) -> Session:
    session = requests.Session()
    session.mount(
        "https://",
        HTTPAdapter(
            max_retries=Retry(
                total=3,
                status_forcelist=[429, 500, 502, 503, 504],
                allowed_methods=["GET", "HEAD"],
                backoff_factor=1,
            )
        ),
    )
    if headers is None:
        headers = {
            "Accept": "text/html",
            "Accept-Encoding": "gzip, deflate, br",
            "Accept-Language": "en-GB,en-US;q=0.9,en;q=0.8",
            "upgrade-insecure-requests": "1",
            "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_6) "
            "AppleWebKit/537.36 (KHTML, like Gecko) "
            "Chrome/90.0.4430.93 Safari/537.36",
        }
    if add is not None:
        headers.update(add)
    session.headers = headers
    return session

def get(
    url, description=None, stream=False, add: dict = None, headers: dict = None
) -> Response:
    session = get_session(add=add, headers=headers)
    logger.info(
        f"Fetching"
        f"{f' {description} from' if description is not None else ''} "
        f"{url}"
    )
    try:
        response = session.get(url, stream=stream)
        response.raise_for_status()
        return response
    except HTTPError as e:
        logger.error(f"HTTP error: {e}")
        return Response()
    except Exception as e:
        logger.error(f"Other error: {e}")
        return Response()

def exists(url, add: dict = None, headers: dict = None) -> bool:
    session = get_session(add=add, headers=headers)
    logger.debug(f"Checking if {url} exists")
    try:
        response = session.head(url)
        response.raise_for_status()
        if response.is_redirect:
            return False
        return response.status_code == 200
    except HTTPError as e:
        logger.error(f"HTTP error: {e}")
        return False
    except Exception as e:
        logger.error(f"Other error: {e}")
        return False


import time

import dramatiq

from challenge.application import repository  # noqa:F401
from challenge.application import app, projections, translator

@dramatiq.actor(queue_name="manual-translation")
def projections_task(id):
    """ Task for updating the read-model of an aggregate.

    As the read-model doesn't need to be updated in real-time, a worker
    is scheduled to do these updates when possible.

    Args:
        id (string): An aggregate UUID4 string.
    """
    projections.update(id)

@dramatiq.actor(queue_name="manual-translation")
def translation_task(id):
    """ Task for processing a translation.

    This task is responsible for requesting the translation service to
    translate a given text from English to Spanish.

    The Translation aggregate is already created and persisted, so this
    task must fetch its data. If for some reason the application goes
    down, it's possible to reprocess "requested" translations adding
    their IDs to the workers queue.

    Args:
        id (string): The translation aggregate UUID4 string.
    """
    translation = repository.get(id)
    translation = translator.process(translation)
    repository.save(translation)
    projections_task.send(id)

@dramatiq.actor(queue_name="machine-translation")
def nmt_task(id):
    """ Task for automatically processing a translation.

    This task is responsible for requesting the Marian-NMT server to
    translate a given text from English to Spanish.

    The Translation aggregate is already created and persisted, so this
    task must fetch its data. If for some reason the application goes
    down, it's possible to reprocess "requested" translations adding
    their IDs to the workers queue.

    Args:
        id (string): The translation aggregate UUID4 string.
    """
    translation = repository.get(id)
    translation = translator.nmt_process(translation)
    repository.save(translation)
    projections_task.send(id)


#! /usr/bin/env python3

import sys
import os
import re
import readline
import cmd2
import argparse

import pentesting_lab.core.global_vars as global_vars
import pentesting_lab.core.vmconfig
import pentesting_lab.core.vminstance
import pentesting_lab.core.vde2
import pentesting_lab.core.session
import pentesting_lab.core.presentation
import pentesting_lab.utils.vagrant_utils
from pentesting_lab.utils.user_input import prompt as user_input

current_session = pentesting_lab.core.session.Session()
vminstances = list()
vde_switches = list()

def show_available_boxes():
    f = open(global_vars.available_boxes_path,'r')
    print("Available boxes:")
    avail_boxes = f.read().split('\n')
    for line in avail_boxes:
        print(line)
    return avail_boxes

def update_vms_names():
    vms_list = pentesting_lab.utils.vagrant_utils.list_vms()
    vms_names = pentesting_lab.utils.vagrant_utils.list_vms_by_name(vms=vms_list)

def start():
    if current_session.load():
        vde_switches = current_session.vde_configs
        print("Current session:")
        print(current_session)
    update_vms_names()
    app = Cli()
    app.load_modules('modules')
    app.cmdloop()

# TODO add history file to preferences.rc
history_file = os.path.expanduser(global_vars.history_file_path)
if not os.path.exists(history_file):
    with open(history_file, "w") as fobj:
        fobj.write("")
readline.read_history_file(history_file)
#TODO add atexit module
#atexit.register(readline.write_history_file, history_file)

class Cli(cmd2.Cmd):
    prompt = "> "
    intro="""
 ________________
< Pentesting Lab >
 ----------------
        \   ^__^
         \  (oo)\_______
            (__)\       )\/\\
                ||----w |
                ||     ||
"""
    modules = {}

    def __init__(self):
        cmd2.Cmd.__init__(self)
        self.iface_cmd_strs = ['create','remove','list']
        self.vm_cmd_strs = ['create','remove','list']
        self.session_cmd_strs = ['create','save','load','show','start','destroy']
        self.exit_cmd_strs = ['maintaining','destroying']

    def postloop(self):
        self.do_exit('destroying')

#### modules loading ####
    def _addloadedmodule(self, module_name, module_handle):
        self.modules[module_name] = module_handle

    def _delloadedmodule(self, module_name):
        del self.modules[module_name]

    def getloadedmodules(self):
        return(list(self.modules.keys()))

    def load_modules(self, modules_dir):
        for root, _, files in os.walk(modules_dir, topdown=False):
            for name in files:
                if name[-1] == 'c' or name[0] == '_':
                    continue
                self.load_module(root.replace('/','.') + "." + name[:-3])

    def load_module(self, module_name):
        gobj = self
        gcls = gobj.__class__
        handle = __import__(module_name)
        modules = [ i for i in dir(handle) if i[0] != '_']
        module = module_name.split('.')[1]
        modulehandle = getattr(handle, module)
        classnames = [ i for i in dir(modulehandle) if i[0] != '_']
        for cls in classnames:
            classhandle = getattr(modulehandle, cls)
            methodnames = [ i for i in dir(classhandle) if i[0] != '_']
            for method in methodnames:
                # Check for method override
                try:
                    getattr(gcls, method)
                    # is there any better way?
                    if method is "cmd2" or method is "argparse":
                        raise AttributeError
                    else:
                        raise(Exception("Override is not allowed (method:"+module_name+":"+method+")"))
                except AttributeError:
                    pass

                methodhandle = getattr(classhandle, method)
                setattr(gcls, method, methodhandle)
            gobj._addloadedmodule(cls, classhandle)

    def unload_module(self, module_name):
        gobj = self
        gcls = self.__class__

        classhandle = self.modules[module_name]
        methodnames = [ i for i in dir(classhandle) if i[0] != '_']
        for method in methodnames:
            methodhandle = getattr(classhandle, method)
            delattr(gcls, method)
        gobj._delloadedmodule(module_name)

#### iface methods ####
    iface_parser = argparse.ArgumentParser()
    iface_parser.add_argument('cmd', help='= (create|remove|list)')
    iface_parser.add_argument('name', nargs='?', help='= name of the iface')

    @cmd2.with_argparser(iface_parser)
    def do_iface(self,args):
        """Manage network interfaces"""
        if args.cmd == 'create':
            print("--- Creating net interface ---")
            name = user_input(choice_msg='Enter iface name:')
            sniff = user_input(choice_msg='Sniff (true|false):',boolean=True)
            switch = pentesting_lab.core.vde2.VdeSwitch(name,sniff)
            switch.create_switch()
            current_session.add_iface(switch)
            vde_switches.append(switch)
            print('Created iface {}\n'.format(name))
        elif args.cmd == 'remove':
                current_session.remove_iface(args.name)
                for i, o in enumerate(vde_switches):
                    if o.name == args.name:
                        vde_switches[i].halt_switch()
                        del vde_switches[i]
                        break
        elif args.cmd == 'list':
            pentesting_lab.core.presentation.print_network_interfaces(current_session)

    def complete_iface(self, text, line, begidx, endidx):
        index_dict = \
            {
                1: self.iface_cmd_strs,
                2: [n.name for n in current_session.vde_configs]
            }
        return self.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)

#### vm methods ####
    vm_parser = argparse.ArgumentParser()
    vm_parser.add_argument('cmd', help='= (create|remove|list)')
    vm_parser.add_argument('name', nargs='?',help='= name of the vm')

    @cmd2.with_argparser(vm_parser)
    def do_vm(self,args):
        """Manage virtual machines"""
        if args.cmd == 'create':
            print("--- Creating machine ---")
            name = user_input(choice_msg="Enter name of vm: ")
            avail_boxes = show_available_boxes()
            box = user_input(choice_msg="Enter name of the box: ", choices=avail_boxes)
            #TODO: check on cpus and memory
            cpus = user_input(choice_msg="Enter the number of cpus: ",digit=True)
            memory = user_input(choice_msg="Enter the memory to allocate: ",digit=True)
            network_interfaces = user_input(choice_msg="Enter network interfaces: ", ret_list=True,
                    choices=[i.name for i in current_session.vde_configs], choices_only=True, accept_none=True)
            packages = user_input(choice_msg="Enter packages: ", ret_list=True, accept_none=True)
            playbooks = user_input(choice_msg="Enter playbooks: ", ret_list=True, accept_none=True)
            #TODO: better parsing, multiple select
            config = pentesting_lab.core.vmconfig.VMConfig(name=name, box=box, cpus=int(cpus),
                    memory=int(memory), gui=False, linked_clone=False,network_interfaces=network_interfaces,
                    packages=packages, playbooks=playbooks)
            current_session.add_vm(config)
            print('Created vm {}\n'.format(name))
        elif args.cmd == 'remove':
            current_session.remove_vm(args.name)
            for i, o in enumerate(vminstances):
                if o.config.name == args.name:
                    del vminstances[i]
                    break
        elif args.cmd == 'list':
            pentesting_lab.core.presentation.print_vms_general(current_session)

    def complete_vm(self, text, line, begidx, endidx):
        index_dict = \
            {
                1: self.vm_cmd_strs,
                2: [n.name for n in current_session.vm_configs]
            }
        return self.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)

#### session methods ####
    session_parser = argparse.ArgumentParser()
    session_parser.add_argument('cmd', help='= (create|save|load|start|stop|destroy)')

    @cmd2.with_argparser(session_parser)
    def do_session(self,args):
        """Manage the session"""

        if args.cmd == 'create':
            # TODO
            pass
        elif args.cmd == 'load':
            print("Session file path (default ./sessions/session.cfg):", end="")
            path = input()
            if not path.strip():
                path = global_vars.sessions_path
            try:
                current_session.load(path=path)
                vde_switches = current_session.vde_configs
            except FileNotFoundError:
                pass
            #TODO
        elif args.cmd == 'save':
            current_session.save()
            #TODO
        elif args.cmd == 'show':
            current_session.show()
            #TODO
        elif args.cmd == 'start':
            for vmconfig in current_session.vm_configs:
                vminstances.append(pentesting_lab.core.vminstance.GuestMachine(vmconfig))
            for vm in vminstances:
                vm.setup()
                vm.up()
        elif args.cmd == 'stop':
            #TODO
            for vm in vminstances:
                vm.halt()
        elif args.cmd == 'destroy':
            #TODO
            pass

    def complete_session(self, text, line, begidx, endidx):
        index_dict = \
            {
                1: self.session_cmd_strs
            }
        return self.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)

#### ssh methods ####
    ssh_parser = argparse.ArgumentParser()
    ssh_parser.add_argument('host', help='the host to ssh in')

    @cmd2.with_argparser(ssh_parser)
    def do_ssh(self,args):
        """SSH commands"""
        for vm in vminstances:
            if vm.config.name == args.host:
                vm.ssh()

    def complete_ssh(self, text, line, begidx, endidx):
        index_dict = \
            {
                1: [n.config.name for n in vminstances]
            }
        return self.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)

#### exit methods ####
    exit_parser = argparse.ArgumentParser()
    exit_parser.add_argument('mode', nargs='?', default='maintaining', help='= destroying if you want to destroy all the file created and kill all the processes')

    @cmd2.with_argparser(exit_parser)
    def do_exit(self,args):
        """EXIT"""
        if args.mode == 'destroying':
            for vm in vminstances:
                vm.cleanup(destroy=True)
        elif args.mode == 'maintaining':
            for vm in vminstances:
                vm.cleanup(destroy=False)

        for switch in current_session.vde_configs:
            switch.halt_switch()
        sys.exit()

    def complete_exit(self, text, line, begidx, endidx):
        index_dict = \
            {
                1: self.exit_cmd_strs
            }
        return self.index_based_complete(text, line, begidx, endidx, index_dict=index_dict)

##TODO playbooks methods


from bs4 import BeautifulSoup
import re
from datetime import date
import urllib.parse as urlparse

class HTMLParser:
    def _getListURL(self, url, html) -> str:
        soup = BeautifulSoup(html, "html.parser", from_encoding="utf-8")
        divs = soup.find_all('div', class_='cont_nav')
        try:
            for div in divs:
                if re.search(r'学术动态', div.find('h3').get_text()) is None:
                    continue
                lecture_list_link = div.find(
                    'a', href=re.compile(r'newListLogic.shtml.+'))
                lecture_list_url = lecture_list_link['href']
                lecture_list_url = urlparse.urljoin(url, lecture_list_url)
                return lecture_list_url
        except:
            pass
        return None

    def _getAllListPages(self, url, html) -> {}:
        soup = BeautifulSoup(html, "html.parser", from_encoding="utf-8")
        try:
            text = soup.find('td', text=re.compile(r'共.+')).get_text()
            num = int(re.search(r'共1/(\d*)页', text).group(1))
            return {url + '&pageNumber=' + str(i) for i in range(1, num+1)}
        except:
            return None

    def _getLectureURLs(self, url, html) -> {}:
        soup = BeautifulSoup(html, "html.parser", from_encoding="utf-8")
        links = soup.find_all('a', href=re.compile(r'newsDetails_zw.shtml.+'))
        lecture_urls = set()
        for link in links:
            if re.search(r'讲座', link.get_text()) is not None:
                lecture_urls.add(urlparse.urljoin(url, link['href']))
        return lecture_urls

    def _getLectureDetails(self, url, html) -> {}:
        soup = BeautifulSoup(html, "html.parser", from_encoding="utf-8")
        text = re.sub(r'<[^<>]>', '', soup.get_text())
        text = re.sub(r'\xa0', '', text)
        try:
            page_title = soup.find_all('h3')[1].get_text()
        except:
            page_title = '未识别'
        try:
            title = re.search(r'(题目|主题)(：|:)(\r|\n)*([^\r\n]*)', text).group(4)
        except:
            title = '未识别'
        try:
            speaker = re.search(
                r'(报 ?告 ?人|演 ?讲 ?者|主 ?讲 ?人)(：|:)(\r|\n)*([^\r\n]*)', text).group(4)
        except:
            speaker = '未识别'
        try:
            temp = re.search(r'发布时间：\[(\d+)-(\d+)-(\d+)\]', text)
            announce_date = date(int(temp.group(1)), int(
                temp.group(2)), int(temp.group(3)))
        except:
            announce_date = date(1, 1, 1)
        try:
            time = re.search(
                r'(?<!发布)时间(：|:)(\r|\n)*([^\r\n]*)', text).group(3)
        except:
            time = '未识别'
        try:
            room = re.search(r'地点(：|:)(\r|\n)*([^\r\n]*)', text).group(3)
        except:
            room = '未识别'
        return {'url': url, 'page title': page_title, 'title': title, 'speaker': speaker,
                'time': time, 'room': room, 'announce date': announce_date}

    def parse(self, url, html) -> ():
        if url is None or html is None:
            return None
        elif re.search(r'newsDetails_zw.shtml', url) is not None:
            data = self._getLectureDetails(url, html)
            return (data, None)
        elif re.search(r'newListLogic.shtml.+&pageNumber', url) is not None:
            lecture_urls = self._getLectureURLs(url, html)
            return (None, lecture_urls)
        elif re.search(r'newListLogic.shtml', url) is not None:
            all_list_urls = self._getAllListPages(url, html)
            return (None, all_list_urls)
        elif re.search(r'index.shtml', url) is not None:
            list_url = self._getListURL(url, html)
            return (None, {list_url})
        else:
            return None


"""
Cross Validation

Author: Mark Tierney
Last Edited: 10/1/16

Using cross validation to determine ideal bandwidth for kernel density estimates.
"""
# Import usual packages
import numpy as np
import astroML as astro
import matplotlib.pyplot as plt
from scipy.spatial import cKDTree
from astroML.plotting import hist
from sklearn.neighbors import KernelDensity
from scipy.stats import ks_2samp
from scipy.stats import mannwhitneyu

# import U-R data from Eco survey
data = np.genfromtxt("ECO_DR1_withradec.csv", delimiter=",", dtype=None, names=True)
ur_color = data['MODELU_RCORR']

binWidth_knuth = astro.density_estimation.knuth_bin_width(ur_color) # determine knuth bin widths for full sample

# Set up regular array of bandwidths
bdwidth = np.linspace(.01, binWidth_knuth, 10)

# create random training set (%50), test (25%), and cross val. (25%) sets
mask = np.zeros(np.size(ur_color), dtype=bool)
mask[np.random.choice(np.size(ur_color),size=(np.size(ur_color))/2,replace=False)]=True
ur_training = ur_color[mask]
ur_other = ur_color[~mask]
mask2 = np.zeros((np.size(ur_color)-1)/2, dtype=bool)
mask2[np.random.choice((np.size(ur_color)-1)/2,size=(np.size(ur_color)-1)/4,replace=False)]=True
ur_crossVal = ur_other[mask2]
ur_test = ur_other[~mask2]

# Kolmogroov-Smirnov to test whether random samples provide similar distributions
print('Kolmogorov-Smirnov for training set and other set p-value: %0.3f' % ks_2samp(ur_training, ur_other)[1])
print('Kolmogorov-Smirnov for test and cross val. sets p-value: %0.3f' % ks_2samp(ur_crossVal, ur_test)[1])

sum_log_like = np.zeros(len(bdwidth))

for i in range(len(bdwidth)):
    kde_train = KernelDensity(kernel='gaussian', bandwidth=bdwidth[i]).fit(ur_training[:,np.newaxis])
    kde_samples = kde_train.score_samples(ur_training[:,np.newaxis])

    kde_crossVal = KernelDensity(kernel='gaussian', bandwidth=.5).fit(kde_samples[:,np.newaxis])
    kde_samples_crossVal = kde_train.score_samples(ur_crossVal[:,np.newaxis])

    sum_log_like[i] = (1./len(ur_color))*np.sum(np.log(np.exp(kde_samples_crossVal)))

opt_band_ind = (sum_log_like.tolist()).index(np.max(sum_log_like))
opt_bdwidth = bdwidth[opt_band_ind]

print('Optimum bandwidth by cross-val: %0.3f' % opt_bdwidth)

# to assess error, calculating KDE for test and training sets
kde_train_err = KernelDensity(kernel='gaussian', bandwidth=opt_bdwidth).fit(ur_training[:,np.newaxis])
kde_test_err = KernelDensity(kernel='gaussian', bandwidth=opt_bdwidth).fit(ur_test[:,np.newaxis])
kde_samples_train_err = kde_train.score_samples(ur_training[:,np.newaxis])
kde_samples_test_err = kde_train.score_samples(ur_test[:,np.newaxis])

plt.figure(0)
plt.clf()
plt.plot(ur_training[:,np.newaxis],np.exp(kde_samples_train_err),'r.',label='KDE Opt. Bandwidth (Training)')
plt.plot(ur_test[:,np.newaxis],np.exp(kde_samples_test_err),'b.',label='KDE Opt. Bandwidth (Test)')
plt.title('Error Assessment Plot')
plt.legend(loc='upper left')

# Swapping Sets to test variation
sum_log_like = np.zeros(len(bdwidth))

for i in range(len(bdwidth)):
    kde_train = KernelDensity(kernel='gaussian', bandwidth=bdwidth[i]).fit(ur_other[:,np.newaxis])
    kde_samples = kde_train.score_samples(ur_other[:,np.newaxis])

    kde_crossVal = KernelDensity(kernel='gaussian', bandwidth=.5).fit(kde_samples[:,np.newaxis])
    kde_samples_crossVal = kde_train.score_samples(ur_test[:,np.newaxis])

    sum_log_like[i] = (1./len(ur_color))*np.sum(np.log(np.exp(kde_samples_crossVal)))

opt_band_ind = (sum_log_like.tolist()).index(np.max(sum_log_like))
opt_bdwidth = bdwidth[opt_band_ind]

print('Optimum bandwidth by cross-val: %0.3f' % opt_bdwidth)

# to assess error, calculating KDE for test and training sets
kde_train_err = KernelDensity(kernel='gaussian', bandwidth=opt_bdwidth).fit(ur_other[:,np.newaxis])
kde_test_err = KernelDensity(kernel='gaussian', bandwidth=opt_bdwidth).fit(ur_crossVal[:,np.newaxis])
kde_samples_train_err = kde_train.score_samples(ur_other[:,np.newaxis])
kde_samples_test_err = kde_train.score_samples(ur_crossVal[:,np.newaxis])

plt.plot(ur_other[:,np.newaxis],np.exp(kde_samples_train_err),'g.',label='KDE Opt. Bandwidth (Training Flipped)')
plt.plot(ur_test[:,np.newaxis],np.exp(kde_samples_test_err),'k.',label='KDE Opt. Bandwidth (Test Flipped)')
plt.title('Error Assessment Plot')
plt.legend(loc='upper left')

# There is not much variation between the graphs when the random samples that compose
# our test, cross validation, and training sets are all swapped. This is desired as
# they should all be drawing from the same initial distribution and we'd expect the
# training set to produce the same density esimate with little variation depending
# on its training set.


import pandas as pd
df = pd.read_csv('https://archive.ics.uci.edu/ml/machine-learning-databases/breast-cancer-wisconsin/wdbc.data', header=None)

# Assign Features and Label Output
from sklearn.preprocessing import LabelEncoder
X = df.loc[:, 2:].values
y = df.loc[:, 1].values
le = LabelEncoder()
y = le.fit_transform(y)

# Splitting Test and Train Data
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.20, random_state=1)

# Using Pipeline to Aggregate several steps
from sklearn.preprocessing import StandardScaler
from sklearn.decomposition import PCA
from sklearn.linear_model import LogisticRegression
from sklearn.pipeline import Pipeline
pipe_lr = Pipeline([('scl', StandardScaler()), ('pca', PCA(n_components=2)), ('clf', LogisticRegression(random_state=1))])
pipe_lr.fit(X_train, y_train)
print('Test Accuracy: %.3f' % pipe_lr.score(X_test, y_test))

# K-fold cross validation
import numpy as np
from sklearn.cross_validation import StratifiedKFold
kfold = StratifiedKFold(y=y_train, n_folds=10, random_state=1)
scores = []
for k, (train, test) in enumerate(kfold):
    pipe_lr.fit(X_train[train], y_train[train])
    score = pipe_lr.score(X_train[test], y_train[test])
    scores.append(score)
    print('Fold: %s, Class dist.: %s, Acc: %.3f' % (k+1, np.bincount(y_train[train]), score))

print('CV accuracy: %.3f +/- %.3f' % ( np.mean(scores), np.std(scores)))

# Using K-fold validation with pipeline
from sklearn.cross_validation import cross_val_score
scores = cross_val_score(estimator=pipe_lr, X=X_train, y=y_train, cv=10, n_jobs=1)
print('CV accuracy scores: %s' % scores)
print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

# Plot Learning Curve
import matplotlib.pyplot as plt
from sklearn.learning_curve import learning_curve
pipe_lr = Pipeline(
    [
        ('scl', StandardScaler()),
        ('clf', LogisticRegression(penalty='l2', random_state=0))
    ]
)
train_sizes, train_scores, test_scores =\
        learning_curve(estimator=pipe_lr,
                       X=X_train,
                       y=y_train,
                       train_sizes=np.linspace(0.1, 1.0, 10),
                       cv=10,
                       n_jobs=1)
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.plot(train_sizes, train_mean,
         color='blue', marker='o',
         markersize=5,
         label='training accuracy')
plt.fill_between(train_sizes,
                 train_mean + train_std,
                 train_mean - train_std,
                 alpha=0.15, color='blue')
plt.plot(train_sizes, test_mean,
         color='green', linestyle='--',
         marker='s', markersize=5,
         label='validation accuracy')
plt.fill_between(train_sizes,
                 test_mean + test_std,
                 test_mean - test_std,
                 alpha=0.15, color='green')
plt.grid()
plt.xlabel('Number of training samples')
plt.ylabel('Accuracy')
plt.legend(loc='lower right')
plt.ylim([0.8, 1.0])
plt.show()

# Plot Validation Curve
from sklearn.learning_curve import validation_curve
param_range = [0.001, 0.01, 0.1, 1.0, 10.0, 100.0]
train_scores, test_scores = validation_curve(
                estimator=pipe_lr,
                 X=X_train,
                 y=y_train,
                 param_name='clf__C',
                 param_range=param_range,
                 cv=10)
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.plot(param_range, train_mean,
          color='blue', marker='o',
          markersize=5,
          label='training accuracy')
plt.fill_between(param_range, train_mean + train_std,
                  train_mean - train_std, alpha=0.15,
                  color='blue')
plt.plot(param_range, test_mean,
          color='green', linestyle='--',
          marker='s', markersize=5,
          label='validation accuracy')
plt.fill_between(param_range,
                  test_mean + test_std,
                  test_mean - test_std,
                  alpha=0.15, color='green')
plt.grid()
plt.xscale('log')
plt.legend(loc='lower right')
plt.xlabel('Parameter C')
plt.ylabel('Accuracy')
plt.ylim([0.8, 1.0])
plt.show()

# Using Grid Search to perform hyperparameter tuning
from sklearn.grid_search import GridSearchCV
from sklearn.svm import SVC
pipe_svc = Pipeline([('scl', StandardScaler()),
                      ('clf', SVC(random_state=1))])
param_range = [0.0001, 0.001, 0.01, 0.1, 1.0, 10.0, 100.0, 1000.0]
param_grid = [{'clf__C': param_range,
                'clf__kernel': ['linear']},
               {'clf__C': param_range,
                'clf__gamma': param_range,
                'clf__kernel': ['rbf']}]
gs = GridSearchCV(estimator=pipe_svc,
                   param_grid=param_grid,
                   scoring='accuracy',
                   cv=10,
                   n_jobs=-1)
gs = gs.fit(X_train, y_train)
print(gs.best_score_)
print(gs.best_params_)

clf = gs.best_estimator_
clf.fit(X_train, y_train)
print('Test accuracy: %.3f' % clf.score(X_test, y_test))

# Comparing different Classifier with grid search and nested k-fold cross validation
# SVM
gs = GridSearchCV(estimator=pipe_svc,
                   param_grid=param_grid,
                   scoring='accuracy',
                   cv=2,
                   n_jobs=-1) # to tune paramenter
scores = cross_val_score(gs, X_train, y_train, scoring='accuracy', cv=5) # Train with optimal params
print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

# Decision Tree Classifier
from sklearn.tree import DecisionTreeClassifier
gs = GridSearchCV(
       estimator=DecisionTreeClassifier(random_state=0),
       param_grid=[
            {'max_depth': [1, 2, 3, 4, 5, 6, 7, None]}],
       scoring='accuracy',
       cv=5)
scores = cross_val_score(gs, X_train, y_train, scoring='accuracy', cv=2)
print('CV accuracy: %.3f +/- %.3f' % (np.mean(scores), np.std(scores)))

# Different Performance Metrics
# Confusion Matrix
from sklearn.metrics import confusion_matrix
pipe_svc.fit(X_train, y_train)
y_pred = pipe_svc.predict(X_test)
confmat = confusion_matrix(y_true=y_test, y_pred=y_pred)
print(confmat)

fig, ax = plt.subplots(figsize=(2.5, 2.5))
ax.matshow(confmat, cmap=plt.cm.Blues, alpha=0.3)
for i in range(confmat.shape[0]):
     for j in range(confmat.shape[1]):
         ax.text(x=j, y=i,
                 s=confmat[i, j],
                 va='center', ha='center')
plt.xlabel('predicted label')
plt.ylabel('true label')
plt.show()

# Precision, Recall, F1 Score
from sklearn.metrics import precision_score
from sklearn.metrics  import recall_score, f1_score
print('Precision: %.3f' % precision_score(y_true=y_test, y_pred=y_pred))
print('Recall: %.3f' % recall_score(y_true=y_test, y_pred=y_pred))
print('F1: %.3f' % f1_score(y_true=y_test, y_pred=y_pred))


import numba
import numpy as np

class VoxelGenerator(object):
    """Voxel generator in numpy implementation.
    Args:
        voxel_size (list[float]): Size of a single voxel
        point_cloud_range (list[float]): Range of points
    """
    def __init__(self,
                 voxel_size,
                 point_cloud_range):

        point_cloud_range = np.array(point_cloud_range, dtype=np.float32)
        voxel_size = np.array(voxel_size, dtype=np.float32)
        grid_size = (point_cloud_range[3:] - point_cloud_range[:3]) / voxel_size
        grid_size = np.round(grid_size).astype(np.int64)

        self._voxel_size = voxel_size
        self._point_cloud_range = point_cloud_range
        self._grid_size = grid_size

    def generate(self, points):
        """Generate voxels given points."""
        return points_to_voxel(points, self._voxel_size, self._point_cloud_range, True)

    @property
    def voxel_size(self):
        """list[float]: Size of a single voxel."""
        return self._voxel_size

    @property
    def point_cloud_range(self):
        """list[float]: Range of point cloud."""
        return self._point_cloud_range

    @property
    def grid_size(self):
        """np.ndarray: The size of grids."""
        return self._grid_size

    def __repr__(self):
        """str: Return a string that describes the module."""
        repr_str = self.__class__.__name__
        indent = ' ' * (len(repr_str) + 1)
        repr_str += f'(voxel_size={self._voxel_size},\n'
        repr_str += indent + 'point_cloud_range='
        repr_str += f'{self._point_cloud_range.tolist()},\n'
        repr_str += indent + f'grid_size={self._grid_size.tolist()}'
        repr_str += ')'
        return repr_str

def points_to_voxel(points,
                    voxel_size,
                    coors_range,
                    reverse_index=True):
    """convert kitti points(N, >=3) to voxels.
    Args:
        points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and
            points[:, 3:] contain other information such as reflectivity.
        voxel_size (list, tuple, np.ndarray): [3] xyz, indicate voxel size
        coors_range (list[float | tuple[float] | ndarray]): Voxel range.
            format: xyzxyz, minmax
        reverse_index (bool): Whether return reversed coordinates.
            if points has xyz format and reverse_index is True, output
            coordinates will be zyx format, but points in features always
            xyz format.
    Returns:
        tuple[np.ndarray]:
            coordinates: [M, 3] int32 tensor.
            point_voxel_ids: [N] int32 tensor.
    """
    if not isinstance(voxel_size, np.ndarray):
        voxel_size = np.array(voxel_size, dtype=points.dtype)
    if not isinstance(coors_range, np.ndarray):
        coors_range = np.array(coors_range, dtype=points.dtype)
    voxelmap_shape = (coors_range[3:] - coors_range[:3]) / voxel_size
    voxelmap_shape = tuple(np.round(voxelmap_shape).astype(np.int32).tolist())
    if reverse_index:
        voxelmap_shape = voxelmap_shape[::-1]
    coor_to_voxelidx = -np.ones(shape=voxelmap_shape, dtype=np.int32)
    coors = np.zeros(shape=(np.prod(voxelmap_shape), 3), dtype=np.int32)
    point_voxel_ids = -np.ones(shape=(points.shape[0]), dtype=np.int32)
    if reverse_index:
        voxel_num = _points_to_voxel_reverse_kernel(points, voxel_size, coors_range, coor_to_voxelidx,
                                                    coors, point_voxel_ids)

    else:
        voxel_num = _points_to_voxel_kernel(points, voxel_size, coors_range, coor_to_voxelidx,
                                            coors, point_voxel_ids)

    coors = coors[:voxel_num]
    return coors, point_voxel_ids

@numba.jit(nopython=True)
def _points_to_voxel_reverse_kernel(points,
                                    voxel_size,
                                    coors_range,
                                    coor_to_voxelidx,
                                    coors,
                                    point_voxel_ids):
    """convert kitti points(N, >=3) to voxels.
    Args:
        points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and
            points[:, 3:] contain other information such as reflectivity.
        voxel_size (list, tuple, np.ndarray): [3] xyz, indicate voxel size
        coors_range (list[float | tuple[float] | ndarray]): Range of voxels.
            format: xyzxyz, minmax
        coor_to_voxelidx (np.ndarray): A voxel grid of shape (D, H, W),
            which has the same shape as the complete voxel map. It indicates
            the index of each corresponding voxel.
        coors (np.ndarray): Created coordinates of each voxel.
        point_voxel_ids (np.ndarray):  Created voxel index of each point.
    Returns:
        tuple[np.ndarray]:
            coordinates: Shape [M, 3].
            point_voxel_ids: Shape [N].
    """
    # put all computations to one loop.
    # we shouldn't create large array in main jit code, otherwise
    # reduce performance
    N = points.shape[0]
    # ndim = points.shape[1] - 1
    ndim = 3
    ndim_minus_1 = ndim - 1
    grid_size = (coors_range[3:] - coors_range[:3]) / voxel_size
    # np.round(grid_size)
    # grid_size = np.round(grid_size).astype(np.int64)(np.int32)
    grid_size = np.round(grid_size, 0, grid_size).astype(np.int32)
    coor = np.zeros(shape=(3, ), dtype=np.int32)
    voxel_num = 0
    failed = False
    for i in range(N):
        failed = False
        for j in range(ndim):
            c = np.floor((points[i, j] - coors_range[j]) / voxel_size[j])
            if c < 0 or c >= grid_size[j]:
                failed = True
                break
            coor[ndim_minus_1 - j] = c
        if failed:
            continue
        voxelidx = coor_to_voxelidx[coor[0], coor[1], coor[2]]
        if voxelidx == -1:
            voxelidx = voxel_num
            voxel_num += 1
            coor_to_voxelidx[coor[0], coor[1], coor[2]] = voxelidx
            coors[voxelidx] = coor
        point_voxel_ids[i] = voxelidx
    return voxel_num

@numba.jit(nopython=True)
def _points_to_voxel_kernel(points,
                            voxel_size,
                            coors_range,
                            coor_to_voxelidx,
                            coors,
                            point_voxel_ids):
    """convert kitti points(N, >=3) to voxels.
    Args:
        points (np.ndarray): [N, ndim]. points[:, :3] contain xyz points and
            points[:, 3:] contain other information such as reflectivity.
        voxel_size (list, tuple, np.ndarray): [3] xyz, indicate voxel size.
        coors_range (list[float | tuple[float] | ndarray]): Range of voxels.
            format: xyzxyz, minmax
        coor_to_voxelidx (np.ndarray): A voxel grid of shape (D, H, W),
            which has the same shape as the complete voxel map. It indicates
            the index of each corresponding voxel.
        coors (np.ndarray): Created coordinates of each voxel.
        point_voxel_ids (np.ndarray):  Created voxel index of each point.
    Returns:
        tuple[np.ndarray]:
            coordinates: Shape [M, 3].
            point_voxel_ids: Shape [N].
    """
    N = points.shape[0]
    # ndim = points.shape[1] - 1
    ndim = 3
    grid_size = (coors_range[3:] - coors_range[:3]) / voxel_size
    # grid_size = np.round(grid_size).astype(np.int64)(np.int32)
    grid_size = np.round(grid_size, 0, grid_size).astype(np.int32)

    # lower_bound = coors_range[:3]
    # upper_bound = coors_range[3:]
    coor = np.zeros(shape=(3, ), dtype=np.int32)
    voxel_num = 0
    failed = False
    for i in range(N):
        failed = False
        for j in range(ndim):
            c = np.floor((points[i, j] - coors_range[j]) / voxel_size[j])
            if c < 0 or c >= grid_size[j]:
                failed = True
                break
            coor[j] = c
        if failed:
            continue
        voxelidx = coor_to_voxelidx[coor[0], coor[1], coor[2]]
        if voxelidx == -1:
            voxelidx = voxel_num
            voxel_num += 1
            coor_to_voxelidx[coor[0], coor[1], coor[2]] = voxelidx
            coors[voxelidx] = coor
        point_voxel_ids[i] = voxelidx
    return voxel_num


import sys
import pymongo
from nltk.tokenize import TreebankWordTokenizer as Tokenizer
from nltk.stem import WordNetLemmatizer
from nltk.stem.lancaster import LancasterStemmer
from nltk.corpus import stopwords

def lemmatize_keywords(db):
    tokenizer = Tokenizer()
    lemmatizer = WordNetLemmatizer()
    stemmer = LancasterStemmer()
    keyword_collections = ['keywords', 'keywords_not_of_interest']
    for kw_collection in keyword_collections:
        keywords = db[kw_collection].find({})
        for kw in keywords:
            tokens = [t for t in tokenizer.tokenize(kw['_id']) if t not in stopwords.words('english')]
            lemma_keywords_str = " ".join([lemmatizer.lemmatize(token) for token in tokens])
            stem_keywords_str = " ".join([stemmer.stem(token) for token in tokens])
            db[kw_collection].update({'_id': kw['_id']},
                    {
                        '$set': {'lemma': lemma_keywords_str, 'stem': stem_keywords_str}
                    }
                )
        db[kw_collection].create_index([("lemma", pymongo.ASCENDING)])
        db[kw_collection].create_index([("stem", pymongo.ASCENDING)])

if __name__ == "__main__":
    try:
        client = pymongo.MongoClient("localhost", 27017)
        db = client.msag
        lemmatize_keywords(db)
    except:
      print >> sys.stderr, "Error:", sys.exc_info()


from collections import deque

data = deque(input().split())

colours = {'red', 'yellow', 'blue', 'orange', 'purple', 'green'}
req_colours = {
    'orange': {'red', 'yellow'},
    'purple': {'red', 'blue'},
    'green': {'yellow', 'blue'},
}

result = []

while data:
    first_data = data.popleft()
    second_data = data.pop() if data else ''

    for colour in (first_data + second_data, second_data + first_data):
        if colour in colours:
            result.append(colour)
            break
    else:
        for element in (first_data[:-1], second_data[:-1]):
            if element:
                data.insert(len(data) // 2, element)

for colour in set(req_colours.keys()).intersection(result):
    if not req_colours[colour].issubset(result):
        result.remove(colour)

print(result)


from flask import Blueprint, request
from op import image_op as io
image=Blueprint("image",__name__)

'''
根据图片获取字符串
'''
@image.route('/image_translation',methods=['POST'])
def image_translation():
    image = request.form.get('image')
    name = request.form.get('name')
    print(name)
    if not (image and name):
        return 'no data'
    return io.image_translation_op(name,image)


from abc import ABC, abstractmethod
from typing import Optional

from state import State

class Solver(ABC):
    @staticmethod
    @abstractmethod
    def solve(state: State, order: str) -> Optional[State]:
        if not state.is_solvable():
            raise Exception("Not solvable")
        if len(order) != 4 or 'L' not in order or 'R' not in order or 'U' not in order or 'D' not in order:
            raise Exception("Invalid search order")

        return


from sklearn.mixture import GaussianMixture
from sklearn.cluster import KMeans
import numpy as np
import matplotlib.pyplot as plt

dataset = np.loadtxt("result1.txt", dtype=float)
dataset = dataset/1000
X = dataset.reshape(5,1000).T
kmeans = KMeans(2, random_state=0)
sse = {}

# for k in range(1, 10):
#     kmeans = KMeans(n_clusters=k, max_iter=1000).fit(X)
#     #X["clusters"] = kmeans.labels_
#     #print(data["clusters"])
#     sse[k] = kmeans.inertia_ # Inertia: Sum of distances of samples to their closest cluster center
# plt.figure()
# plt.plot(list(sse.keys()), list(sse.values()))
# plt.xlabel("Number of cluster")
# plt.ylabel("SSE")
# plt.show()

model = kmeans.fit(X)
labels = model.predict(X)

acc = model.score(X)
print(acc)
plt.scatter(X[:, 0], X[:, 1], c=labels)
plt.show()

# gmm = GaussianMixture(n_components=5,covariance_type='diag').fit(X)
# labels1 = gmm.predict(X)
# plt.scatter(X[:, 0], X[:, 1], c=labels1, s=40, cmap='viridis');
# plt.show()


import numpy as np
from scipy.optimize import linear_sum_assignment

from ...utils._param_validation import StrOptions, validate_params
from ...utils.validation import check_array, check_consistent_length

__all__ = ["consensus_score"]

def _check_rows_and_columns(a, b):
    """Unpacks the row and column arrays and checks their shape."""
    check_consistent_length(*a)
    check_consistent_length(*b)
    checks = lambda x: check_array(x, ensure_2d=False)
    a_rows, a_cols = map(checks, a)
    b_rows, b_cols = map(checks, b)
    return a_rows, a_cols, b_rows, b_cols

def _jaccard(a_rows, a_cols, b_rows, b_cols):
    """Jaccard coefficient on the elements of the two biclusters."""
    intersection = (a_rows * b_rows).sum() * (a_cols * b_cols).sum()

    a_size = a_rows.sum() * a_cols.sum()
    b_size = b_rows.sum() * b_cols.sum()

    return intersection / (a_size + b_size - intersection)

def _pairwise_similarity(a, b, similarity):
    """Computes pairwise similarity matrix.

    result[i, j] is the Jaccard coefficient of a's bicluster i and b's
    bicluster j.

    """
    a_rows, a_cols, b_rows, b_cols = _check_rows_and_columns(a, b)
    n_a = a_rows.shape[0]
    n_b = b_rows.shape[0]
    result = np.array(
        [
            [similarity(a_rows[i], a_cols[i], b_rows[j], b_cols[j]) for j in range(n_b)]
            for i in range(n_a)
        ]
    )
    return result

@validate_params(
    {
        "a": [tuple],
        "b": [tuple],
        "similarity": [callable, StrOptions({"jaccard"})],
    },
    prefer_skip_nested_validation=True,
)
def consensus_score(a, b, *, similarity="jaccard"):
    """The similarity of two sets of biclusters.

    Similarity between individual biclusters is computed. Then the
    best matching between sets is found using the Hungarian algorithm.
    The final score is the sum of similarities divided by the size of
    the larger set.

    Read more in the :ref:`User Guide <biclustering>`.

    Parameters
    ----------
    a : tuple (rows, columns)
        Tuple of row and column indicators for a set of biclusters.

    b : tuple (rows, columns)
        Another set of biclusters like ``a``.

    similarity : 'jaccard' or callable, default='jaccard'
        May be the string "jaccard" to use the Jaccard coefficient, or
        any function that takes four arguments, each of which is a 1d
        indicator vector: (a_rows, a_columns, b_rows, b_columns).

    Returns
    -------
    consensus_score : float
       Consensus score, a non-negative value, sum of similarities
       divided by size of larger set.

    References
    ----------

    * Hochreiter, Bodenhofer, et. al., 2010. `FABIA: factor analysis
      for bicluster acquisition
      <https://www.ncbi.nlm.nih.gov/pmc/articles/PMC2881408/>`__.
    """
    if similarity == "jaccard":
        similarity = _jaccard
    matrix = _pairwise_similarity(a, b, similarity)
    row_indices, col_indices = linear_sum_assignment(1.0 - matrix)
    n_a = len(a[0])
    n_b = len(b[0])
    return matrix[row_indices, col_indices].sum() / max(n_a, n_b)


import cv2
import numpy as np

filepath = './aruco_photo/'
markersX = 1  # X轴上标记的数量
markersY = 1  # Y轴上标记的数量
markerLength = 100  # 标记的长度，单位是像素
markerSeparation = 20  # 每个标记之间的间隔，单位像素
margins = markerSeparation  # 标记与边界之间的间隔
borderBits = 10  # 标记的边界所占的bit位数
showImage = True

width = markersX * (markerLength + markerSeparation) - markerSeparation + 2 * margins
height =markersY * (markerLength + markerSeparation) - markerSeparation + 2 * margins

dictionary = cv2.aruco.Dictionary_get(cv2.aruco.DICT_6X6_250)  # 生成标记的字典ID
board = cv2.aruco.GridBoard_create(markersX, markersY, float(markerLength), float(markerSeparation), dictionary)
print(cv2.aruco_GridBoard.getGridSize(board))
img = cv2.aruco_GridBoard.draw(board, (500, 500), 1)
cv2.imwrite(filepath + '1.png', img)


"""Measures for chaotic dynamical systems."""
from __future__ import annotations

from typing import Callable

import numpy as np

def largest_lyapunov_exponent(
    iterator_func: Callable[[np.ndarray], np.ndarray],
    starting_point: np.ndarray,
    deviation_scale: float = 1e-10,
    steps: int = int(1e3),
    part_time_steps: int = 15,
    steps_skip: int = 50,
    dt: float = 1.0,
    initial_pert_direction: np.ndarray | None = None,
    return_convergence: bool = False,
) -> float | np.ndarray:
    """Numerically calculate the largest lyapunov exponent given an iterator function.

    See: Sprott, Julien Clinton, and Julien C. Sprott. Chaos and time-series analysis.
    Vol. 69. Oxford: Oxford university press, 2003.

    Args:
        iterator_func: Function to iterate the system to the next time step:
                       x(i+1) = F(x(i))
        starting_point: The starting_point of the main trajectory.
        deviation_scale: The L2-norm of the initial perturbation.
        steps: Number of renormalization steps.
        part_time_steps: Time steps between renormalization steps.
        steps_skip: Number of renormalization steps to perform, before tracking the log
                    divergence. Avoid transients by using steps_skip.
        dt: Size of time step.
        initial_pert_direction:
            - If np.ndarray: The direction of the initial perturbation.
            - If None: The direction of the initial perturbation is assumed to be
                np.ones(..).
        return_convergence: If True, return the convergence of the largest LE; a numpy
                            array of the shape (N, ).

    Returns:
        The largest Lyapunov Exponent. If return_convergence is True: The convergence
        (np.ndarray), else just the float value, which is the last value in the
        convergence.
    """
    x_dim = starting_point.size

    if initial_pert_direction is None:
        initial_pert_direction = np.ones(x_dim)

    initial_perturbation = initial_pert_direction * (
        deviation_scale / np.linalg.norm(initial_pert_direction)
    )

    log_divergence = np.zeros(steps)

    x = starting_point
    x_pert = starting_point + initial_perturbation

    for i_n in range(steps + steps_skip):
        for i_t in range(part_time_steps):
            x = iterator_func(x)
            x_pert = iterator_func(x_pert)
        dx = x_pert - x
        norm_dx = np.linalg.norm(dx)
        x_pert = x + dx * (deviation_scale / norm_dx)
        if i_n >= steps_skip:
            log_divergence[i_n - steps_skip] = np.log(norm_dx / deviation_scale)

    if return_convergence:
        return np.array(
            np.cumsum(log_divergence) / (np.arange(1, steps + 1) * dt * part_time_steps)
        )
    else:
        return float(np.average(log_divergence) / (dt * part_time_steps))


import multiprocessing
from time_track import time_track
from volatility_sort_and_print import *

def volatility_count(file_name):
    ticker = None
    max_price = 0
    min_price = 0

    with open(file_name, 'r', encoding='utf8') as file:
        file.readline()
        for line in file:
            sec_id, trade_time, price, quantity = line[:-1].split(',')
            price = float(price)
            if sec_id != ticker:
                ticker = sec_id
                max_price, min_price = price, price
            elif max_price < price:
                max_price = price
            elif min_price > price:
                min_price = price

    half_sum = (max_price + min_price) / 2
    volatility = round((((max_price - min_price) / half_sum) * 100), 2)

    return ticker, volatility

@time_track
def main(path):
    pool = multiprocessing.Pool(multiprocessing.cpu_count())
    vol_list = pool.map(volatility_count, path_to_file(path=path))
    pool.close()
    pool.join()
    sort(dict(vol_list))

if __name__ == '__main__':
    path = 'trades'
    main(path=path)


from instructions import Instruction
from config import LINE_DEFAULT_CHARACTER

class Line(Instruction):
    def __init__(self, x1, y1, x2, y2, filling_character=LINE_DEFAULT_CHARACTER):
        Instruction.__init__(self, filling_character)
        self.x1 = x1
        self.y1 = y1
        self.x2 = x2
        self.y2 = y2

    @staticmethod
    def validate(args):
        if len(args) < 4:
            raise ValueError
        numbers = []
        for i in range(0, 4):
            numbers.append(int(args[i]))
        return Line(numbers[0], numbers[1], numbers[2], numbers[3])

    def __str__(self):
        return "Line({}, {}, {}, {})".format(self.x1, self.y1, self.x2, self.y2)


from django.forms import ModelForm
from .models import Post, Comment

class PostForm(ModelForm):
    def __init__(self, *args, **kwargs):
        super(PostForm, self).__init__(*args, **kwargs)
        self.fields['group'].empty_label = None
        self.fields['group'].widget.choices = self.fields['group'].choices

    class Meta:
        model = Post
        fields = ('text', 'group', 'image')
        labels = {
            'text': 'Текст поста',
            'group': 'Группа',
        }

class CommentForm(ModelForm):
    class Meta:
        model = Comment
        fields = ('text',)


from typing import Optional

from tortuga.cli.tortugaCli import TortugaCli
from tortuga.exceptions.componentNotFound import ComponentNotFound
from tortuga.wsapi.kitWsApi import KitWsApi

class KitCli(TortugaCli):
    """
    Base kit command line interface class.
    """

    def parseArgs(self, usage: Optional[str] = None):
        kit_attr_group = _('Kit Attribute Options')

        self.addOptionGroup(
            kit_attr_group, _('Kit name/version must be specified.'))

        self.addOptionToGroup(
            kit_attr_group, '--name', help=_('kit name'))

        self.addOptionToGroup(
            kit_attr_group, '--version', help=_('kit version'))

        self.addOptionToGroup(
            kit_attr_group, '--iteration', help=_('kit iteration'))

        self.getParser().add_argument('kitspec', nargs='?')

        return super().parseArgs(usage=usage)

    def get_name_version_iteration(self):
        return self.getKitNameVersionIteration(self.getArgs().kitspec)

    def getKitNameVersionIteration(self, pkgname):
        if pkgname:
            a = pkgname.split('-')

            name = a[0]
            version = None
            iteration = None

            if len(a) == 3:
                version = '-'.join(a[1:-1])
                iteration = a[-1]
            elif len(a) == 2:
                version = a[1]
        else:
            name = self.getArgs().name
            version = self.getArgs().version
            iteration = self.getArgs().iteration

        return name, version, iteration

    def getComponent(self, kitName, kitVersion, kitIteration, compdescr): \
            # pylint: disable=no-self-use
        """
        Get Component matching 'compdescr' from specified kit

        Raises:
            ComponentNotFound
        """

        kitApi = self.configureClient(KitWsApi)

        k = kitApi.getKit(kitName, kitVersion, kitIteration)

        for c in k.getComponentList():
            _compDescr = '%s-%s' % (c.getName(), c.getVersion())

            if _compDescr == compdescr:
                break
        else:
            raise ComponentNotFound(
                'Component [%s] not found in kit [%s]' % (compdescr, k))

        return c


from django.test import TestCase, Client
from django.urls import reverse

from forms.models import User, FormResponse, INVESTIGATION_ROLES
from forms.tests.factories import FormResponseFactory, FormInstanceFactory, TagFactory, UserFactory, \
    InvestigationFactory, FormFactory

def url_for_response(form_response):
    form = form_response.form_instance.form
    return reverse("response_json_edit", kwargs=dict(investigation_slug=form.investigation.slug,
                                                     form_slug=form.slug,
                                                     response_id=form_response.id))

class FormReponseListViewTest(TestCase):
    def setUp(self):
        owner = UserFactory.create()
        self.investigation = InvestigationFactory.create()
        self.investigation.add_user(owner, INVESTIGATION_ROLES.OWNER)

        self.form = FormFactory(investigation=self.investigation)
        self.client.force_login(owner)

    def make_singe_step_instance(self, properties):
        return FormInstanceFactory.create(form_json=[
            {
                "schema":
                    {
                        "title": "first",
                        "slug": "first",
                        "properties": properties
                    }
            }
        ], form=self.form)

    def test_edit_fails_for_viewer(self):
        form_instance = self.make_singe_step_instance({"name": {"type": "string"}})
        form_response = FormResponseFactory.create(json={"name": "Gödel"}, form_instance=form_instance)

        viewer = UserFactory.create()
        self.investigation.add_user(viewer, "V")
        self.client.force_login(viewer)

        response = self.client.post(url_for_response(form_response), data={"json__name": "Escher"})
        self.assertEqual(response.status_code, 403)

        updated_response = FormResponse.objects.get(id=form_response.id)
        self.assertEqual(updated_response.json["name"], "Gödel")

    def test_edit(self):
        form_instance = self.make_singe_step_instance({"name": {"type": "string"}})
        form_response = FormResponseFactory.create(json={"name": "Gödel"}, form_instance=form_instance)

        response = self.client.post(url_for_response(form_response), data={"json__name": "Escher"})
        self.assertEqual(response.status_code, 302)

        updated_response = FormResponse.objects.get(id=form_response.id)
        self.assertEqual(updated_response.json["name"], "Escher")

    def test_edit_non_existing_field(self):
        properties = {"name": {"type": "string"}}
        form_instance = self.make_singe_step_instance(properties)
        form_response = FormResponseFactory.create(json={"name": "Gödel"}, form_instance=form_instance)

        response = self.client.post(url_for_response(form_response), data={"json__color": "Red"})
        self.assertEqual(response.status_code, 400)

        updated_response = FormResponse.objects.get(id=form_response.id)
        self.assertDictEqual(updated_response.json, {"name": "Gödel"})

    def test_add_new_field_to_response(self):
        properties = {"name": {"type": "string"}, "color": {"type": "string"}}
        form_instance = self.make_singe_step_instance(properties)
        form_response = FormResponseFactory.create(json={"name": "Gödel"}, form_instance=form_instance)

        response = self.client.post(url_for_response(form_response), data={"json__color": "Red"})
        self.assertEqual(response.status_code, 302)

        updated_response = FormResponse.objects.get(id=form_response.id)
        self.assertDictEqual(updated_response.json, {"name": "Gödel", "color": "Red"})

    def test_with_json_schema(self):
        properties = {"email": {"type": "string", "format": "email"}}
        form_instance = self.make_singe_step_instance(properties)
        form_response = FormResponseFactory.create(json={"email": "test@example.com"}, form_instance=form_instance)

        response = self.client.post(url_for_response(form_response), data={"json__email": "not an email address"})
        self.assertEqual(response.status_code, 400)
        self.assertEqual(response.content, b"'not an email address' is not a 'email'")

        updated_response = FormResponse.objects.get(id=form_response.id)
        self.assertDictEqual(updated_response.json, {"email": "test@example.com"})


import numpy as np
import csv
from sklearn.neural_network import BernoulliRBM, MLPClassifier
import plot
from rbm import error, percent_error, reformat_data, gen_even_slices

def make_rbm(data, components=50, n_iter=20):
    model = BernoulliRBM(n_components=components, learning_rate=0.1, batch_size=10, n_iter=n_iter,
    	random_state=None, verbose=0)
    model = model.fit(data)
    print("error: {}".format(percent_error(model.gibbs(data), data)))
    return model

def make_dbn(data, components=[150,50], iterations=None):
    if iterations == None:
        iterations = [1]*len(components)
    result = []
    for size, n_iter in zip(components, iterations):
        rbm = make_rbm(data, components=size, n_iter=n_iter)
        result.append(rbm)
        data = np.round(rbm._mean_hiddens(data)).astype(int)
    return result

def run_together(data, layers):
    for layer in layers:
        data = layer._sample_hiddens(data, layer.random_state_)
    for layer in reversed(layers):
        data = layer._sample_visibles(data, layer.random_state_)
    return data

def read_to_array(inf):
    data = []
    with open(inf) as f:
    	reader = csv.reader(f)
    	for row in reader:
    		data.append(np.array([int(_) for _ in row]))
    return np.array(data)

if __name__ == "__main__":
    train_data = read_to_array('binMNIST_data\\bindigit_trn.csv')
    test_data = read_to_array('binMNIST_data\\bindigit_tst.csv')
    train_class = read_to_array('binMNIST_data\\targetdigit_trn.csv')
    test_class = read_to_array('binMNIST_data\\targetdigit_tst.csv')
    print("train data: {}, test data: {}".format(len(train_data), len(test_data)))

    dbn = make_dbn(train_data, [150, 100], [10, 100, 500])
    mlp = MLPClassifier(solver='lbfgs', alpha=1e-5, hidden_layer_sizes=(784, 512), random_state=1)
    mlp.fit(train_data, train_class)

    print("sample / sample error: {}".format(percent_error(dbn[0]._sample_hiddens(train_data, dbn[0].random_state_), dbn[0]._sample_hiddens(train_data, dbn[0].random_state_))))
    print("rounded mean / sample error: {}".format(percent_error(np.round(dbn[0]._mean_hiddens(train_data)).astype(int), dbn[0]._sample_hiddens(train_data, dbn[0].random_state_))))

    #for i in [27, 4, 18, 1, 3, 2, 15, 10, 7, 6]:
    #    #plot.plot_heatmap(reformat_data(test_data[i-1]))
    #    #plot.plot_heatmap(reformat_data(dbn[0].gibbs(test_data[i-1])))
    #    plot.plot_heatmap(reformat_data(run_together(test_data[i-1], dbn)))

    result = run_together(test_data, dbn)
    rbm_result = dbn[0].gibbs(test_data)
    print("first layer total error: {}".format(percent_error(rbm_result, test_data)))
    print("total error: {}".format(percent_error(result, test_data)))
    print(sum([1 if a == b else 0 for a, b in zip(mlp.predict(result), test_class)])/len(result))
    print(sum([1 if a == b else 0 for a, b in zip(mlp.predict(rbm_result), test_class)])/len(rbm_result))