from messageTemplates.msgBody import MsgBody
from spade.template import Template
from spade.message import Message
from datetime import datetime
from typing import Optional, Union
from aioxmpp import JID

def CONTAINER_ARRIVAL_CFP_TEMPLATE(thread: Optional[str] = None) -> Template:
    """
    Template for container arrival CFP message.

    Args:
        thread (str, optional): Thread ID used for accepting only given conversation.

    Accepts:
        - ContainerArrivalCFPMsgBody
    """
    t = Template(thread=thread)
    t.set_metadata("performative", "cfp")
    t.set_metadata("topic", "container_arrival")
    return t

def CONTAINER_ARRIVAL_REFUSE_TEMPLATE(thread: Optional[str] = None) -> Template:
    """
    Template for container arrival refuse message.

    Args:
        thread (str, optional): Thread ID used for accepting only given conversation.

    Accepts:
        - ContainerArrivalRefuseMsgBody
    """
    t = Template(thread=thread)
    t.set_metadata("performative", "refuse")
    t.set_metadata("topic", "container_arrival")
    return t

def CONTAINER_ARRIVAL_PROPOSE_TEMPLATE(thread: Optional[str] = None) -> Template:
    """
    Template for container arrival propose message.

    Args:
        thread (str, optional): Thread ID used for accepting only given conversation.

    Accepts:
        - ContainerArrivalProposeMsgBody
    """
    t = Template(thread=thread)
    t.set_metadata("performative", "propose")
    t.set_metadata("topic", "container_arrival")
    return t

def CONTAINER_ARRIVAL_REJECT_PROPOSAL_TEMPLATE(
    thread: Optional[str] = None,
) -> Template:
    """
    Template for container arrival reject proposal message.

    Args:
        thread (str, optional): Thread ID used for accepting only given conversation.

    Accepts:
        - ContainerArrivalRejectProposalMsgBody
    """
    t = Template(thread=thread)
    t.set_metadata("performative", "reject-proposal")
    t.set_metadata("topic", "container_arrival")
    return t

def CONTAINER_ARRIVAL_ACCEPT_PROPOSAL_TEMPLATE(
    thread: Optional[str] = None,
) -> Template:
    """
    Template for container arrival accept proposal message.

    Args:
        thread (str, optional): Thread ID used for accepting only given conversation.

    Accepts:
        - ContainerArrivalAcceptProposalMsgBody
    """
    t = Template(thread=thread)
    t.set_metadata("performative", "accept-proposal")
    t.set_metadata("topic", "container_arrival")
    return t

class ContainerArrivalCFPMsgBody(MsgBody):
    def __init__(self, container_ids: list[str], date: datetime):
        """
        Container arrival CFP message body.

        Args:
            container_ids (list[str]): List of arriving container IDs.
            date (datetime): Expected arrival date.

        Response:
            - ContainerArrivalProposeMsgBody
            - ContainerArrivalRefuseMsgBody
        """
        self.container_ids = container_ids
        if isinstance(date, datetime):
            date = date.isoformat()
        self.date = date

    def create_message(self, to: Union[JID, str], reply_by: Union[datetime, str], thread: str) -> Message:
        msg = super().create_message(to, reply_by=reply_by, thread=thread)
        msg.set_metadata("performative", "cfp")
        msg.set_metadata("topic", "container_arrival")
        return msg

class ContainerArrivalRefuseMsgBody(MsgBody):
    def __init__(self):
        """Refusal of container arrival CFP."""
        super().__init__()

    def create_message(self, to: Union[JID, str], thread: str) -> Message:
        msg = super().create_message(to, thread=thread)
        msg.set_metadata("performative", "refuse")
        msg.set_metadata("topic", "container_arrival")
        return msg

class ContainerArrivalProposeMsgBody(MsgBody):
    def __init__(self, cost: float, container_count: int):
        """
        Proposes for CFP of container arrival.

        Args:
            cost (float): Best cost of container arrival.
            container_count (int): Number of containers that can be handled.

        Response:
            - ContainerArrivalAcceptProposalMsgBody
            - ContainerArrivalRejectProposalMsgBody
        """
        self.cost = cost
        self.container_count = container_count

    def create_message(self, to: Union[JID, str], reply_by: Union[datetime, str],  thread: str) -> Message:
        msg = super().create_message(to, reply_by=reply_by, thread=thread)
        msg.set_metadata("performative", "propose")
        msg.set_metadata("topic", "container_arrival")
        return msg

class ContainerArrivalRejectProposalMsgBody(MsgBody):
    def __init__(self):
        """Rejects proposal of container arrival."""
        super().__init__()

    def create_message(self, to: Union[JID, str], thread: str):
        msg = super().create_message(to, thread=thread)
        msg.set_metadata("performative", "reject-proposal")
        msg.set_metadata("topic", "container_arrival")
        return msg

class ContainerArrivalAcceptProposalMsgBody(MsgBody):
    def __init__(self):
        """Accepts proposal of container arrival."""
        super().__init__()

    def create_message(self, to: Union[JID, str], thread: str):
        msg = super().create_message(to, thread=thread)
        msg.set_metadata("performative", "accept-proposal")
        msg.set_metadata("topic", "container_arrival")
        return msg


import accelerate
import fire
import torch
import transformers
from torch.utils.data import DataLoader
from tqdm.auto import tqdm

import gate.data.medical.segmentation.automated_cardiac_diagnosis as acd
import gate.data.medical.segmentation.medical_decathlon as md
from gate.models.task_adapters.medical_semantic_segmentation import logger
from gate.models.task_specific_models.semantic_segmentation.timm import (
    build_gate_model,
)

logger.setLevel("DEBUG")

def sub_batch_generator(batch_dict, sub_batch_size):
    """
    Generator function to yield sub-batches from a given batch dictionary.

    Parameters:
    - batch_dict (dict): Dictionary containing original batch data. Each key maps to a tensor or list.
    - sub_batch_size (int): Size of each sub-batch to be returned.

    Yields:
    - dict: Dictionary containing sub-batch data.
    """
    batch_size = None

    # Validate input and get original batch size
    for key, value in batch_dict.items():
        # print(f"key: {key}, value.shape: {value.shape}")
        if batch_size is None:
            batch_size = value.shape[0] * value.shape[1]
        elif batch_size != value.shape[0] * value.shape[1]:
            raise ValueError(
                f"Batch sizes for different keys in batch_dict must be the same. Mismatch at key: {key}"
            )

    # Generate and yield sub-batches
    for start_idx in range(0, batch_size, sub_batch_size):
        end_idx = min(start_idx + sub_batch_size, batch_size)
        sub_batch = {}

        for key, value in batch_dict.items():
            sub_batch[key] = value.reshape(-1, *value.shape[2:])[
                start_idx:end_idx
            ]

        yield sub_batch

def build_dataloader(
    dataset_name,
    data_dir,
    image_size,
    target_image_size,
    transforms,
    batch_size=1,
    num_workers=12,
):
    if dataset_name == "md":
        data = md.build_gate_dataset(
            data_dir=data_dir,
            image_size=image_size,
            label_image_size=target_image_size,
            transforms=transforms,
        )
    elif dataset_name == "acdc":
        data = acd.build_gate_dataset(
            data_dir=data_dir,
            image_size=image_size,
            target_image_size=target_image_size,
            transforms=transforms,
        )
    else:
        raise ValueError("Invalid dataset name")

    return DataLoader(
        data["val"],
        batch_size=batch_size,
        shuffle=True,
        num_workers=num_workers,
    )

def integrate_output_list(output_list):
    def accumulate_outputs(output_accumulator, output_dict):
        for key, value in output_dict.items():
            if isinstance(value, dict):
                if key not in output_accumulator:
                    output_accumulator[key] = {}
                accumulate_outputs(output_accumulator[key], value)
            else:
                if isinstance(value, torch.Tensor):
                    if key not in output_accumulator:
                        output_accumulator[key] = []
                    output_accumulator[key].append(value.detach().cpu())

    output_accumulator = {}
    for output_dict in output_list:
        accumulate_outputs(output_accumulator, output_dict)

    def concatenate_tensors(nested_dict):
        for key, value in nested_dict.items():
            if isinstance(value, dict):
                concatenate_tensors(value)
            else:
                if isinstance(value, list) and len(value) > 0:
                    if value[0].dim() == 0:
                        # Handle scalar tensors
                        nested_dict[key] = torch.cat(
                            [v.unsqueeze(0) for v in value], dim=0
                        )
                    else:
                        # Handle non-scalar tensors
                        nested_dict[key] = torch.cat(value, dim=0)

    concatenate_tensors(output_accumulator)
    return output_accumulator

def detach_and_move_to_cpu(obj):
    if isinstance(obj, torch.Tensor):
        return obj.detach().cpu()
    elif isinstance(obj, dict):
        return {
            key: detach_and_move_to_cpu(value) for key, value in obj.items()
        }
    elif isinstance(obj, list):
        return [detach_and_move_to_cpu(element) for element in obj]
    else:
        return obj

def main(
    dataset_name: str = "md",
    data_dir: str = "/data/",
    image_size: int = 512,
    target_image_size: int = 256,
    batch_size: int = 1,
    sub_batch_size: int = 1,
    num_workers: int = 8,
    eval_mode: bool = False,
):
    model_and_transform = build_gate_model(
        timm_model_name="vit_base_patch16_clip_224.laion2b",
        num_classes=100,
        pretrained=True,
        use_temporal_model=True,
        num_channels=3,
    )

    model = model_and_transform.model
    transforms = model_and_transform.transform
    dataloader = build_dataloader(
        dataset_name,
        data_dir,
        image_size,
        target_image_size,
        transforms,
        batch_size,
        num_workers,
    )

    accelerator = accelerate.Accelerator(
        mixed_precision="fp16",
    )
    model = accelerator.prepare(model)
    dataloader = accelerator.prepare(dataloader)
    optimizer = transformers.AdamW(
        model.parameters(), lr=1e-5, weight_decay=0.0
    )
    optimizer = accelerator.prepare(optimizer)

    with tqdm(total=len(dataloader)) as pbar:
        output_list = []
        for input_dict in dataloader:
            optimizer.zero_grad()
            for batch in sub_batch_generator(input_dict, sub_batch_size):
                if eval_mode:
                    with torch.no_grad():
                        output = model.forward(batch)
                        loss = output["image"]["image"]["loss"]
                else:
                    output = model.forward(batch)
                    loss = output["image"]["image"]["loss"]
                    accelerator.backward(loss)

                output = detach_and_move_to_cpu(output)

                output_list.append(output)

            if not eval_mode:
                optimizer.step()
            output = integrate_output_list(output_list)
            loss = output["image"]["image"]["loss"].mean()
            pbar.set_description(f"loss: {loss.item():.4f}")

            pbar.update(1)

if __name__ == "__main__":
    fire.Fire(main)


import httplib2
import urllib
from bs4 import BeautifulSoup

def global_check_used(class_name, bundle_name):
	max = 0
	de = None
	global_search_url = 'http://codesearch.alipay.net/source/search?project=Android_wallet_master&q=%s&n=%s&start=0'
	h = httplib2.Http()
	__str = global_search_url % (urllib.quote(class_name), "")

	resp, content = h.request(__str, "GET")
	soup = BeautifulSoup(content, "html.parser")
	results = soup.find(id="results")
	if not results:
		print('check size not result')
		return de
	sizeSet = results.find_all('p', class_="pagetitle")
	if sizeSet is None or len(sizeSet) is 0:
		print('none result when global search size' + class_name + "--" + bundle_name)
	else:
		for result in sizeSet:
			bold = result.find_all('b')
			if len(bold) > 2:
				max = int(bold[len(bold)-1].text)

	if max is 0:
		dependencySet = results.find_all('td', class_="f")
		if dependencySet is None or len(dependencySet) is 0:
			print('none result when global search ' + class_name + "--" + bundle_name)
		else:
			de = {}
			for dependency in dependencySet:
				a_link = dependency.a
				href = a_link.get('href').split('/')
				if href[4] != bundle_name:
					de[a_link.get_text()] = href[4]
		return de

	else:
		resp, content = h.request(global_search_url % (urllib.quote(class_name), str(max)), "GET")
		soup = BeautifulSoup(content, "html.parser")
		totalResults = soup.find(id="results")
		if not totalResults:
			print('check maxLine not totalResults')
		totalSet = totalResults.find_all('td', class_="f")
		if totalSet is None or len(totalSet) is 0:
			print('none result when global search ' + class_name + "--" + bundle_name)
		else:
			de = {}
			for single in totalSet:
				a_link = single.a
				href = a_link.get('href').split('/')
				if href[4] != bundle_name:
					de[a_link.get_text()] = href[4]
		return de
	return de


import requests
import arrow
from lxml import etree
from io import StringIO
from intsights_model import IntsightsModel
from base_logger import BaseLogger
import os

FMT = '[%(asctime)s %(levelname)s]: %(message)s'
FILE_DIRECTORY = os.path.dirname(__file__)
LOGS_FOLDER = os.path.join(FILE_DIRECTORY, 'logs')

class IntsightsCrawler():
    AUTHOR_NAMES_CONST = ['Guest', 'Unknown', 'Anonymous']
    BASE_URL = 'http://nzxj65x32vh2fkhk.onion/all'
    ALL_ROWS_ELEMENTS = '//section[@id="list"]//div[@class="col-sm-12"]'
    WINDOWS_URL = 'file:///C:/Users/orif/Desktop/StrongholdPaste.htm'
    TITLE_ELEMENT_IN_ROWS = './/h4'
    TEXT_CONTENT_IN_ROWS = './/div[contains(@class,"well-sm")]'
    AUTHOR_AND_DATE_IN_ROWS = './/div[contains(@class,"pre-footer")]'

    def __init__(self):
        path = os.path.join(LOGS_FOLDER, 'logger.log')
        self.logger = BaseLogger().create_logger('crawler_logger', path, FMT)

    def _get_content_from_html(self):
        http_parse = requests.get(self.BASE_URL)
        content_string = http_parse.text
        return content_string

    def _parse_html_content(self, content):
        parser = etree.HTMLParser()
        tree = etree.parse(StringIO(content), parser)
        return tree

    def _fetch_date_and_author(self, element):
        fetched_text = ''
        element_text = element.xpath(self.AUTHOR_AND_DATE_IN_ROWS)[0]
        for x in element_text.itertext():
            fetched_text += x.replace('\n', '').replace('\t', '')
        fetch_splitted = fetched_text.replace(',', '').split(' ')
        author = fetch_splitted[2]
        if author in self.AUTHOR_NAMES_CONST:
            author = 'Anonymous'
        date_fetched = fetch_splitted[6]+'-'+fetch_splitted[5] + \
            '-'+fetch_splitted[4]+' '+fetch_splitted[7]
        date = arrow.get(date_fetched, 'YYYY-MMM-DD HH:mm:ss')
        return author, date

    def _get_elements_from_tree(self, tree):
        content = ''
        list_of_models = []
        html_root = tree.getroot()
        all_rows = html_root.xpath(self.ALL_ROWS_ELEMENTS)
        self.logger.debug('Found {} rows '.format(len(all_rows)-1))
        for element in all_rows:
            response = IntsightsModel()
            got_title = element.xpath(self.TITLE_ELEMENT_IN_ROWS)
            txt_cont = element.xpath(self.TEXT_CONTENT_IN_ROWS)
            if not got_title:
                continue
            title_text = got_title[0].text
            response.title = title_text.replace('\n', '').replace('\t', '')
            for txt in txt_cont[0].itertext():
                content += txt.replace('\n', '').replace('\t', '')
            response.content = content
            author, date = self._fetch_date_and_author(element)
            response.author = author
            response.date = date
            list_of_models.append(response)
        self.logger.debug('Finished Getting all rows - now sorting by date')
        list_of_models.sort(key=lambda model: model.date, reverse=True)
        for paste_model in list_of_models:
            paste_model.date = str(paste_model.date)
        self.logger.debug('Finished all evaluation - return response')
        return list_of_models

    def _get_proxy_for_onion_sites(self):
        session = requests.session()
        session.proxies = {'http':  'socks5h://localhost:9050',
                           'https': 'socks5h://localhost:9050'}
        return session

    def crawl(self):
        self.logger.debug('Getting proxy for onion sites')
        session = self._get_proxy_for_onion_sites()
        self.logger.debug('Dont getting proxy for onion sites')
        self.logger.debug('Getting onion site by request method')
        response = session.get(self.BASE_URL)
        self.logger.debug('Done getting onion site')
        if response.status_code != 200:
            self.logger.error('site problem - got status code != 200  - exit')
            raise Exception('error code : {}'.format(response.status_code))
        content = response.text

        tree = self._parse_html_content(content)
        self.logger.debug('Getting elements from tree')
        response = self._get_elements_from_tree(tree)
        return response

class Testers:
    @staticmethod
    def test_crawl():
        try:
            print('################### TESTING CRAWL  --- ##################')
            i = IntsightsCrawler()
            response = i.crawl()
            print(response)
        except Exception:
            raise Exception('Problem while crawling site')

def main():
    Testers.test_crawl()

if __name__ == '__main__':
    main()


from torch.utils.data.sampler import Sampler
from collections import defaultdict
import copy
import torch
import random
import numpy as np

class RandomActionSampler(Sampler):
    """
    Randomly sample N identities, then for each identity,
    randomly sample K instances, therefore batch size is N*K.
    Args:
    - data_source (list): list of (img_path, pid, camid).
    - num_instances (int): number of instances per identity in a batch.
    - batch_size (int): number of examples in a batch.
    """

    def __init__(self, data_source, num_actions, num_players, num_instances):
        self.data_source = data_source
        self.batch_size = num_actions * num_players * num_instances
        self.num_actions = num_actions
        self.num_players = num_players
        self.num_instances = num_instances

        self.action_dic = defaultdict(set)
        self.pid_dic = defaultdict(list)
        for index, (_, pid, action_idx, _, _, _) in enumerate(self.data_source):
            self.action_dic[action_idx].add(pid)
            self.pid_dic[pid].append(index)

        self.actions = list(self.action_dic.keys())
        self.pids = list(self.pid_dic.keys())

        # estimate number of examples in an epoch
        self.valid_action = 0
        self.valid_action_list = []
        for action_idx in self.action_dic:
            if len(self.action_dic[action_idx]) < self.num_players: continue
            valid_pid = 0
            for pid in self.action_dic[action_idx]:
                if len(self.pid_dic[pid]) >= self.num_instances:
                    valid_pid += 1
                # if len(self.pid_dic[pid]) >= 2:
                #     valid_pid += 1
            if valid_pid < self.num_players: continue
            self.valid_action += 1
            self.valid_action_list.append(action_idx)

        print('num_actions {}, num_players {}, num_instances {}'.format(self.num_actions, self.num_players, self.num_instances))
        print('action nums', self.valid_action, 'epoch nums', self.valid_action // self.num_actions, 'sample num', self.valid_action // self.num_actions * self.batch_size)
        self.length = self.valid_action // self.num_actions * self.batch_size

    def __iter__(self):
        final_idxs = []

        avai_actions = copy.deepcopy(self.valid_action_list)
        random.shuffle(avai_actions)

        action_num = 0
        for action_idx in avai_actions:
            if action_num >= self.valid_action // self.num_actions * self.num_actions: break
            avai_pids = copy.deepcopy(list(self.action_dic[action_idx]))
            random.shuffle(avai_pids)
            action_pid_num = 0
            for pid in avai_pids:
                if action_pid_num >= self.num_players: break
                idxs = copy.deepcopy(self.pid_dic[pid])
                if len(idxs) < self.num_instances: continue
                # if len(idxs) < 2: continue
                if len(idxs) < self.num_instances:
                    idxs = np.random.choice(idxs, size=self.num_instances, replace=True)
                random.shuffle(idxs)
                final_idxs.extend(idxs[:self.num_instances])
                action_pid_num += 1
            action_num += 1

        return iter(final_idxs)

    def __len__(self):
        return self.length

class RandomIdentitySampler(Sampler):
    """
    Randomly sample N identities, then for each identity,
    randomly sample K instances, therefore batch size is N*K.
    Args:
    - data_source (list): list of (img_path, pid, camid).
    - num_instances (int): number of instances per identity in a batch.
    - batch_size (int): number of examples in a batch.
    """

    def __init__(self, data_source, batch_size, num_instances):
        self.data_source = data_source
        self.batch_size = batch_size
        self.num_instances = num_instances
        self.num_pids_per_batch = self.batch_size // self.num_instances
        self.index_dic = defaultdict(list) #dict with list value
        #{783: [0, 5, 116, 876, 1554, 2041],...,}
        for index, (_, pid, _, _) in enumerate(self.data_source):
            self.index_dic[pid].append(index)
        self.pids = list(self.index_dic.keys())

        # estimate number of examples in an epoch
        self.length = 0
        for pid in self.pids:
            idxs = self.index_dic[pid]
            num = len(idxs)
            if num < self.num_instances:
                num = self.num_instances
            self.length += num - num % self.num_instances

    def __iter__(self):
        batch_idxs_dict = defaultdict(list)

        for pid in self.pids:
            idxs = copy.deepcopy(self.index_dic[pid])
            if len(idxs) < self.num_instances:
                idxs = np.random.choice(idxs, size=self.num_instances, replace=True)
            random.shuffle(idxs)
            batch_idxs = []
            for idx in idxs:
                batch_idxs.append(idx)
                if len(batch_idxs) == self.num_instances:
                    batch_idxs_dict[pid].append(batch_idxs)
                    batch_idxs = []

        avai_pids = copy.deepcopy(self.pids)
        final_idxs = []

        while len(avai_pids) >= self.num_pids_per_batch:
            selected_pids = random.sample(avai_pids, self.num_pids_per_batch)
            for pid in selected_pids:
                batch_idxs = batch_idxs_dict[pid].pop(0)
                final_idxs.extend(batch_idxs)
                if len(batch_idxs_dict[pid]) == 0:
                    avai_pids.remove(pid)

        return iter(final_idxs)

    def __len__(self):
        return self.length

# New add by gu
class RandomIdentitySampler_IdUniform(Sampler):
    """
    Randomly sample N identities, then for each identity,
    randomly sample K instances, therefore batch size is N*K.

    Code imported from https://github.com/Cysu/open-reid/blob/master/reid/utils/data/sampler.py.

    Args:
        data_source (Dataset): dataset to sample from.
        num_instances (int): number of instances per identity.
    """
    def __init__(self, data_source, num_instances):
        self.data_source = data_source
        self.num_instances = num_instances
        self.index_dic = defaultdict(list)
        for index, item in enumerate(data_source):
            pid = item[1]
            self.index_dic[pid].append(index)
        self.pids = list(self.index_dic.keys())
        self.num_identities = len(self.pids)

    def __iter__(self):
        indices = torch.randperm(self.num_identities)
        ret = []
        for i in indices:
            pid = self.pids[i]
            t = self.index_dic[pid]
            replace = False if len(t) >= self.num_instances else True
            t = np.random.choice(t, size=self.num_instances, replace=replace)
            ret.extend(t)
        return iter(ret)

    def __len__(self):
        return self.num_identities * self.num_instances


import requests

response = requests.get("https://s3b.astro.ncu.edu.tw/ai_202309/data/q07.data")

splitted = response.content.split(b"\n")
counted = []

for word in splitted:
    if word not in counted:
        print(f"the word {word} appear in the table {splitted.count(word)} times")
        counted.append(word)


# -*- coding: utf-8 -*-
"""
Created on Sat Nov  5 21:50:08 2016

@author: unknown

I have copied this peice of python code from the internet and the credit goes to https://marcobonzanini.com
Anyway, I improved it by making it more reusable module with much more parameters

"""

#%%
import re, string
from nltk.corpus import stopwords

emoticons_str = r"""
    (?:
        [:=;] # Eyes
        [oO\-]? # Nose (optional)
        [D\)\]\(\]/\\OpP] # Mouth
    )"""

url_str = r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-f][0-9a-f]))+'
hashtag_str = r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)"
mention_str = r'(?:@[\w_]+)'
number_str = r'(?:(?:\d+,?)+(?:\.?\d+)?)'

regex_str = [
    emoticons_str,
    r'<[^>]+>', # HTML tags
    mention_str, # @-mentions
    hashtag_str, # hash-tags
    url_str, # URLs
    number_str, # numbers
    #r"(?:[a-z][a-z'\-_]+[a-z])", # words with - and '
    #r"(?:[a-z][a-z\-_]+[a-z])", # words with - and _
    r"(?:[a-z][a-z']+[a-z])", # words with '
    r"(?:&[A-Za-z]+;)", # > < etc.
    r'(?:[\w_]+)', # other words
    r'(?:\S)' # anything else
]

tokens_re = re.compile(r'('+'|'.join(regex_str)+')', re.VERBOSE | re.IGNORECASE)
emoticon_re = re.compile(r'^'+emoticons_str+'$', re.VERBOSE | re.IGNORECASE)
mention_re = re.compile(mention_str+'$', re.VERBOSE | re.IGNORECASE)
number_re = re.compile(number_str+'$', re.VERBOSE | re.IGNORECASE)
url_re = re.compile(url_str+'$', re.VERBOSE | re.IGNORECASE)
hashtag_re = re.compile(hashtag_str+'$', re.VERBOSE | re.IGNORECASE)
html_re = re.compile(r"(?:&[A-Za-z]+;)", re.VERBOSE | re.IGNORECASE)

puncchars = list(string.punctuation)
eng_stopwords = set(stopwords.words('english'))
ily_stopwords = set(stopwords.words('italian'))

def tokenize(s):
    return tokens_re.findall(s)

def preprocess(s, lowercase=True, emoicons=False, numbers=False, mentions=False, url=False, hashtag=False, punctuation=False, return_text=False, stop_words=False):
    tokens = tokenize(s)
    if lowercase:
        tokens = [token if emoticon_re.search(token) else token.lower() for token in tokens]

    tmp = []
    for token in tokens:
        if not emoicons and emoticon_re.search(token):
            continue
        if not punctuation and token in puncchars:
            continue
        if not stop_words and (token in ily_stopwords or token in eng_stopwords):
            continue
        if not url and url_re.search(token):
            continue
        if not hashtag and hashtag_re.search(token):
            continue
        if not mentions and mention_re.search(token):
            continue
        if not numbers and number_re.search(token):
            continue
        if html_re.search(token):
            continue
        if token == 'rt' or token == 'RT' and len(tmp)==0:
            continue

        tmp.append(token)
    tokens = tmp

    if return_text:
        res = ' '.join(tokens)
        return res.strip()

    return tokens

if __name__ == '__main__':
    tweet = """RT @marcobonzanini: Just an  6.5 5.8889 it's you're   #this-is-a-combination  Example!
    :D
    http://example.com #NLP"""
    tweet = "Ayyy you from my hood what school you went to?  > Wanna show you how much I'm dedicated to you.   "
    print(preprocess(tweet, lowercase=False, numbers=True, return_text=True, emoicons=True, mentions=True, url=True, hashtag=True, punctuation=True, stop_words=True))
    print(preprocess(tweet, return_text=True, mentions=False, stop_words=False, hashtag=True))

        #text = text.replace('\t', ' ').replace('\n', ' ')
        ##text = ' '.join(re.sub("(@[A-Za-z0-9]+)|(&[A-Za-z]+;)|(#[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",text).split())
        #text = ' '.join(re.sub("(@[A-Za-z0-9]+)|(&[A-Za-z]+;)|([^0-9A-Za-z# \t])|(\w+:\/\/\S+)"," ",text).split())
        #return text.lower().strip()


import numpy as np
import torch

import deepxde as dde
from . import baseclass
from ..model.fnn import PFNN

class PoissonInv(baseclass.BasePDE):
    def __init__(self):
        super().__init__()
        # output dim
        self.output_config = [{"name": s} for s in ["u", "a"]]
        # geom
        self.bbox = [0, 1] * 2
        self.geom = dde.geometry.Rectangle(xmin=[0, 0], xmax=[1, 1])

        # PDE
        def pde(x, ua):
            u, a = ua[:, 0:1], ua[:, 1:2]
            u_x = dde.grad.jacobian(u, x, i=0, j=0)
            u_y = dde.grad.jacobian(u, x, i=0, j=1)
            d_au = dde.grad.jacobian(a * u_x, x, i=0, j=0) + dde.grad.jacobian(a * u_y, x, i=0, j=1)
            return d_au + self.f_src(x)

        self.pde = pde
        self.set_pdeloss(num=1)

        self.ref_sol = lambda xy: np.concatenate((self.u_ref(xy), self.a_ref(xy)), axis=1)

        self.recommend_net = dde.nn.PFNN(
            [self.input_dim] + 5 * [[50] * self.output_dim] + [self.output_dim], "tanh", "Glorot normal"
        )

        bc_x = np.linspace(0, 1, 50)
        bc_y = np.linspace(0, 1, 50)
        bc_x, bc_y = np.meshgrid(bc_x, bc_y)
        bc_xy = np.stack((bc_x.reshape(-1), bc_y.reshape(-1))).T
        # fmt: off
        self.add_bcs([{
            'component': 0,
            'points': bc_xy,
            'values': self.u_ref(bc_xy) + np.random.normal(loc=0, scale=0.1, size=(2500, 1)),
            'type': 'pointset',
            'name': 'data_loss',
        }, {
            'component': 1,
            'function': self.a_ref,
            'bc': (lambda _, on_boundary: on_boundary),
            'type': 'dirichlet',
            'name': 'bc_a',
        }])
        # fmt: on

        self.training_points()

    @staticmethod
    def a_ref(xy):
        x, y = xy[:, 0:1], xy[:, 1:2]
        return 1 / (1 + x**2 + y**2 + (x - 1) ** 2 + (y - 1) ** 2)

    @staticmethod
    def u_ref(xy):
        x, y = xy[:, 0:1], xy[:, 1:2]
        return np.sin(np.pi * x) * np.sin(np.pi * y)

    @staticmethod
    def f_src(xy):
        x, y = xy[:, 0:1], xy[:, 1:2]
        # fmt: off
        return (
            2 * np.pi**2 * torch.sin(np.pi * x) * torch.sin(np.pi * y) * PoissonInv.a_ref(xy)
            + 2 * np.pi * (
                (2 * x + 1) * torch.cos(np.pi * x) * torch.sin(np.pi * y)
                + (2 * y + 1) * torch.sin(np.pi * x) * torch.cos(np.pi * y)
            ) * PoissonInv.a_ref(xy) ** 2
        )
        # fmt: on

    @staticmethod
    def a_ref_x(xy):
        x, y = xy[:, 0:1], xy[:, 1:2]
        return (2 - 4 * x) / (2 * x**2 - 2 * x + 2 * y**2 - 2 * y + 3) ** 2

    @staticmethod
    def a_ref_y(xy):
        x, y = xy[:, 0:1], xy[:, 1:2]
        return (2 - 4 * y) / (2 * x**2 - 2 * x + 2 * y**2 - 2 * y + 3) ** 2

class HeatInv(baseclass.BaseTimePDE):
    def __init__(self):
        super().__init__()
        # output dim
        self.output_config = [{"name": s} for s in ["u", "a"]]
        # geom
        self.bbox = [-1, 1] * 2 + [0, 1]
        self.geom = dde.geometry.Rectangle(xmin=[-1, -1], xmax=[1, 1])
        timedomain = dde.geometry.TimeDomain(0, 1)
        self.geomtime = dde.geometry.GeometryXTime(self.geom, timedomain)

        def f(xyt):
            x, y, t = xyt[:, 0:1], xyt[:, 1:2], xyt[:, 2:3]
            s, c, p = torch.sin, torch.cos, np.pi
            # fmt: off
            return torch.exp(-t) * (
                (4 * p**2 - 1) * s(p * x) * s(p * y)
                + p**2 * (
                    2 * s(p * x) ** 2 * s(p * y) ** 2
                    - c(p * x) ** 2 * s(p * y) ** 2
                    - s(p * x) ** 2 * c(p * y) ** 2
                )
            )
            # fmt: on

        def u_ref(xyt):
            x, y, t = xyt[:, 0:1], xyt[:, 1:2], xyt[:, 2:3]
            return np.exp(-t) * np.sin(np.pi * x) * np.sin(np.pi * y)

        def a_ref(xyt):  # irrelevent to time domain
            x, y, t = xyt[:, 0:1], xyt[:, 1:2], xyt[:, 2:3]
            return 2 + np.sin(np.pi * x) * np.sin(np.pi * y)

        self.f_src = f
        self.u_ref = u_ref
        self.a_ref = a_ref

        # PDE
        def pde(x, ua):
            u, a = ua[:, 0:1], ua[:, 1:2]
            u_x = dde.grad.jacobian(u, x, i=0, j=0)
            u_y = dde.grad.jacobian(u, x, i=0, j=1)
            u_t = dde.grad.jacobian(u, x, i=0, j=2)
            d_au = dde.grad.jacobian(a * u_x, x, i=0, j=0) + dde.grad.jacobian(a * u_y, x, i=0, j=1)

            return u_t - d_au - f(x)

        self.pde = pde
        self.set_pdeloss(num=1)

        self.ref_sol = lambda xyt: np.concatenate((u_ref(xyt), a_ref(xyt)), axis=1)

        self.recommend_net = PFNN(
            [self.input_dim] + 5 * [[50] * self.output_dim] + [self.output_dim],
            "tanh",
            "Glorot normal",
            split_mask=np.array([[[1, 1, 1]], [[1, 1, 0]]]),
        )

        data_pts = np.loadtxt("ref/heatinv_points.dat")
        # fmt: off
        self.add_bcs([{
            'component': 0,
            'points': data_pts,
            'values': u_ref(data_pts) + np.random.normal(loc=0, scale=0.1, size=(2500, 1)),
            'type': 'pointset',
            'name': 'data_loss',
        }, {
            'component': 1,
            'function': a_ref,
            'bc': (lambda _, on_boundary: on_boundary),
            'type': 'dirichlet',
            'name': 'bc_a',
        }])
        # fmt: on

        self.training_points(mul=4)


import sys, pygame, life

pygame.init()
ALIVE, DEAD = 1,0
alive_count = 0
colors = white, black, grey, dark_grey, blue, red = (255,255,255),(0,0,0),(150,150,150),(110,110,110),(0,180,180), (200,0,0)
sizes = width, height, block_size = 700, 600, 10 # To make adjustable later
screen = pygame.display.set_mode((width,height)) # Create screen display

def create_board(): #  Create the underlying array to be returned
    return life.dead_state(int(height/block_size),int(height/block_size))

def home_view():
    screen.fill(grey)
    draw_grid()
    play_button(False)
    clear_button()
    pygame.display.flip() # Update the display

def play_button(playing):
    if not playing:
        start_button = pygame.Rect(width-95,100,90,45) # Create the start button and draw it
        pygame.draw.rect(screen,dark_grey,start_button)

        reg_font = pygame.font.Font(None, 30)
        start_txt = reg_font.render("START", True, white)
        screen.blit(start_txt,(618,115))

        pygame.display.flip()
    else:
        start_button = pygame.Rect(width-95,100,90,45) # Create the start button and draw it
        pygame.draw.rect(screen,dark_grey,start_button)

        reg_font = pygame.font.Font(None, 30)
        start_txt = reg_font.render("STOP", True, white)
        screen.blit(start_txt,(623,115))

        pygame.display.flip()

def clear_button():
        clear_button = pygame.Rect(width-95,155,90,45)
        pygame.draw.rect(screen,dark_grey,clear_button)
        reg_font = pygame.font.Font(None, 30)
        start_txt = reg_font.render("CLEAR", True, white)
        screen.blit(start_txt,(615,168))
        pygame.display.flip()

def kill_board(board):
    for r in range(len(board)):
        for c in range(len(board[0])):
            board[r][c] = 0
    return board
def draw_grid():
    for y in range(0,height,block_size):
        pygame.draw.line(screen,dark_grey,[0,y],[width-100,y])
    for x in range(0,height,block_size):
        pygame.draw.line(screen,dark_grey,[x+10,0],[x+10,height])

def grid_click(life_board, pos):
    if(life_board[int(pos[1]/block_size)][int(pos[0]/block_size)] == DEAD):
        rect = pygame.Rect(pos[0] - (pos[0]%block_size),pos[1] - (pos[1]%block_size),block_size,block_size)
        pygame.draw.rect(screen,blue,rect)
        draw_grid()
        pygame.display.flip()
        life_board[int(pos[1]/block_size)][int(pos[0]/block_size)] = ALIVE
    elif(life_board[int(pos[1]/block_size)][int(pos[0]/block_size)] == ALIVE): # If ALIVE, make DEAD
        rect = pygame.Rect(pos[0] - (pos[0]%block_size),pos[1] - (pos[1]%block_size),block_size,block_size)
        pygame.draw.rect(screen,grey,rect)
        draw_grid()
        pygame.display.flip()
        life_board[int(pos[1]/block_size)][int(pos[0]/block_size)] = DEAD

def gen_life_board(life_board):
    alive_count=0
    for c in range(0,width-100,block_size):
        for r in range(0,height,block_size):
            rect = pygame.Rect(c,r,block_size,block_size)
            if(life_board[int(r/block_size)][int(c/block_size)]==0):
                colour = grey
            else:
                colour = blue
                alive_count = alive_count + 1
            pygame.draw.rect(screen,colour,rect,0)
    draw_grid()
    pygame.display.flip()
    return(life.next_board_state(life_board))

def play_life():
    home_view()
    board = create_board()
    playing = False
    while True:
        for event in pygame.event.get():
            if(event.type == pygame.QUIT): sys.exit()
            if(event.type == pygame.MOUSEBUTTONDOWN):
                if((0,0) <= event.pos <= (width-100,height) and not playing): # If the grid is clicked before we click start
                    grid_click(board,event.pos)
                elif(100 <= event.pos[1] <= 145): # If start button is clicked
                    playing = not playing
                    play_button(playing)
                elif(155 <= event.pos[1] <= 200 and not playing):
                    gen_life_board(kill_board(board))
                    clear_button()
            if(event.type == pygame.KEYDOWN and event.__dict__['key'] ==13 and not playing):
               board = gen_life_board(board)

        if playing:
            board = gen_life_board(board)

if __name__ == "__main__":
    play_life()


#!/usr/bin/env python

"""Loads tagged sentences. (Do not run main script as it is memory intensive.)"""

import argparse
import nicer
from multiprocessing import Pool
from itertools import repeat
from nltk.probability import FreqDist

from sent_tools import *

"""
Typical usage :
    Imports
"""

def dist_output(dist, output):
    # Writes distribution to output depending in order of frequency
    # Input : distribution, output file
    for x in dist.most_common():
        output.write(str(x[0]) + '  ' + str(x[1]))
        output.write("\n")
    output.write("\n")

def nnp_dist(tagfile, cores):
    # Creates distribution of words that begin sentenes and are taggged NNP
    ids, sents, dist = first_word_filter(tagfile, cores, "NNP")
    return dist

def first_word_filter(fname, cores, tag):
    # Returns list of first words of every sentence if the predicted pos tag is tag
    ids, tag_list = load_tags(fname, cores)
    filtered_ids = []
    filtered_sents = []
    word_dist = FreqDist()
    with Pool(processes=cores) as p:
            for return_tuple in p.starmap(
                check_one_sent_tag,
                zip(ids,
                tag_list,
                repeat(tag),
                ),
                1000,
            ):
                if return_tuple:
                    filtered_ids += return_tuple[0]
                    filtered_sents += return_tuple[1]
                    word_dist.update(return_tuple[2])

    return filtered_ids, filtered_sents, word_dist

def check_one_sent_tag(this_id, this_sent, tag):
    # Checks if first word of the sentence is tagged tag
    if this_sent[0][1] == tag:
        return this_id, this_sent, this_sent[0][1]

def untag_line_to_tokens(line):
    sent_id, sent = split_sentence_id(line)
    tokenized = tokenize(sent)
    tokens = [word.split('_')[0] for word in tokenized]
    return tokens

def load_one_sent_tags(line):
    # Loads one sentence of tags
    sent_id, sent = split_sentence_id(line)
    tokenized = tokenize(sent)
    tags = [tuple(word.split('_')) for word in tokenized]
    return sent_id, tags

def load_tag_lines(lines, cores=5):
    with Pool(processes=cores) as p:
            tags = p.imap(
                load_one_sent_tags,
                lines,
                250,
            )
            all_tags = list(zip(*tags))
            ids = all_tags[0]
            sents = all_tags[1]

    return ids, sents

def load_tags(tagfile, cores=5):
    # Loads tags from file of tagged sentences
    all_tags = []
    with Pool(processes=cores) as p:
            tags = p.imap(
                load_one_sent_tags,
                tagfile.readlines(),
                250,
            )
            all_tags = list(zip(*tags))
            ids = all_tags[0]
            sents = all_tags[1]
    tagfile.close()

    return ids, sents

def is_sent(sent):
    # Checks if the list (sentence) is actually a sentence
    for word in sent:
        if type(word) is not tuple or len(word) != 2:
            return False

    return True

def main(args):
    # DO NOT RUN!
    # Super memory intensive
    dist = nnp_dist(args.file, args.cores)
    dist_output(args, dist)

if __name__ == '__main__':
    nicer.make_nice()
    parser = argparse.ArgumentParser()

    parser.add_argument("--file", "-f",type=argparse.FileType("r"),
                            help="txt file to read proof/tags from")

    parser.add_argument("--output_sentences", "-os", type=argparse.FileType('w'),
                            help="txt file to write tagged sentences to")

    parser.add_argument("--output_wordlist", "-ow", type=argparse.FileType('w'),
                            help="txt file to write wordlist to")

    parser.add_argument( "--cores", "-c",
                            help="Number of cores to use", type=int, default=4)

    parser.add_argument("--tag", "-t", default=None,
                            help="specifies pos tag (given by off the shelf tagger) to filter")

    parser.add_argument("--output", "-o", type=argparse.FileType('w'),
                            help="txt file to write to")

    args = parser.parse_args()

    main(args)
    args.file.close()


import sys
from cnnModelBuilder.src.cnnModelBuilder import CNNModelBuilder
import sys
import logging
import numpy as np

class CNNModelPrediction:
  def __init__(self, cnn_config_file=None, batch_size=128):
    self.cnn_config_file = cnn_config_file
    self.batch_size = batch_size

    self.__loadCNNModel()

  def __loadCNNModel(self):
    self.model = CNNModelBuilder()

    try:
      self.model.loadJSONConfigFile(self.cnn_config_file)
    except FileNotFoundError as e:
      logging.error(("FileNotFoundError in CNNModelPrediction. CNN config file not found, please check both path and filename: %s" % e), exc_info=True)
      raise e

    try:
      self.model.loadModel()
    except FileNotFoundError as e:
      logging.error(("FileNotFoundError in CNNModelPrediction. CNN model file not found, please check both path and filename in configuration file: %s" % e), exc_info=True)
      raise e

  def runPrediction(self, x_batch=None):
    if x_batch is None:
        logging.warning("Warning: x_batch param cannot be 'None' for model prediction. Please check it!")
        return None

    return self.model.cnn_model.predict(np.array(x_batch).astype(np.float32), self.batch_size)


from matplotlib.transforms import BlendedGenericTransform
from matplotlib import patches
from matplotlib.patches import Circle, Arc
import matplotlib.pyplot as plt
plt.rcParams['mathtext.fontset'] = 'cm'
import numpy as np
dtr = np.pi/180

import control

mygray = '#808B96'
grid_dict = {'linestyle':'dashed',\
             'color':mygray, \
             'linewidth':1.0, \
             'zorder':0, \
}

class root_locus_sketch(object):
    def __init__(self, ax=None, \
                 xmax=2, xmin=-5.5, ymax=5.5, ymin=-5.5, \
                 xlims=[-8,4], ylims=[-6,6], \
                 fontdict = {'size': 20, 'family':'serif'}, \
                 axis_lw=2, \
                 ):
        if ax is None:
            fig = plt.figure(figsize=(9,9))
            ax = fig.add_subplot(111)
        self.ax = ax
        self.xmin = xmin
        self.xmax = xmax
        self.ymin = ymin
        self.ymax = ymax
        self.xlims = xlims
        self.ylims = ylims
        self.axis_lw = axis_lw
        self.fontdict = fontdict

    def draw_arrow(self, start_coords, end_coords, \
                   fc='k', ec=None, lw=None, **plot_args):
        if not hasattr(self, 'hw'):
            self.set_arrow_lengths()
        if ec is None:
            ec = fc
        # if lw is None:
        #     lw = self.arrow_lw
        ##start_A = transform_coords(start_coords, HT)
        ##stop_A = transform_coords(end_coords, HT)
        dx = end_coords[0]-start_coords[0]
        dy = end_coords[1]-start_coords[1]

        self.ax.arrow(start_coords[0], start_coords[1], dx, dy, \
                      lw=lw, fc=fc, ec=ec, \
                      width=0.01, \
                      head_width=self.hw, head_length=self.hl, \
                      #overhang = self.ohg, \
                      length_includes_head=True, clip_on = False, \
                      **plot_args)

    def draw_axis(self):
        self.draw_arrow((0,0), (self.xmax,0), \
                        lw=self.axis_lw)
                        #fc=self.axis_color,
        #self.place_rotated_text(self.xmax+self.axis_label_shift, 0, xlabel, HT)
        self.draw_arrow((0,0), (0,self.ymax), \
                        lw=self.axis_lw)
        self.draw_arrow((0,0), (0,self.ymin), \
                        lw=self.axis_lw)
                        #fc=self.axis_color,
        #self.place_rotated_text(0, self.ymax+self.axis_label_shift, ylabel, HT)
        self.draw_arrow((0,0), (self.xmin,0), \
                        lw=self.axis_lw)

    def set_arrow_lengths(self):
        self.hw = 1./30.*(self.ymax-self.ymin)
        self.hl = 1./20.*(self.xmax-self.xmin)
        self.lw = 2. # axis line width
        self.ohg = 0.3 # arrow overhang

    def axis_off(self):
        self.ax.set_axis_off()

    def draw_marker(self, point, style='go', size=12, **plot_kwargs):
        self.ax.plot([np.real(point)],[np.imag(point)], \
                     style, markersize=size, **plot_kwargs)

    def draw_markers(self, point_list, style='go', size=12, **plot_kwargs):
        for point in point_list:
            self.draw_marker(point, style=style, size=size, **plot_kwargs)

    def add_text(self, point, text, xoffset=0, yoffset=0, \
                 fontdict=None):
        if fontdict is None:
            fontdict = self.fontdict
        self.ax.text(np.real(point)+xoffset, np.imag(point)+yoffset, \
                     text, fontdict=fontdict)

    def add_math_text(self, point, text, *args, **kwargs):
        if text[0] != '$':
            text = '$%s$' % text
        self.add_text(point, text, *args, **kwargs)

    def draw_point_with_label(self, point, text, style='go', size=12, \
                          xoffset=0, yoffset=0, \
                          fontdict=None):
        self.draw_marker(point, style=style, size=size)
        self.add_text(point, text, xoffset=xoffset, yoffset=yoffset, \
                      fontdict=fontdict)

    def draw_line(self, start, end, linestyle='k--', **plot_kwargs):
        self.ax.plot([np.real(start), np.real(end)], \
                     [np.imag(start), np.imag(end)], \
                     linestyle, **plot_kwargs)

    def draw_lines_to_point(self, point, pole_or_zero_list, \
                            linestyle='k--', **plot_kwargs):
        for pz in pole_or_zero_list:
            self.draw_line(pz, point, linestyle=linestyle, **plot_kwargs)

    def draw_poles(self, pole_list, style='rX', size=12, **plot_kwargs):
        for i, p in enumerate(pole_list):
            j = i+1
            self.ax.plot([np.real(p)],[np.imag(p)], \
                         style, markersize=size, **plot_kwargs)

    def draw_zeros(self, zero_list, style='ro', size=12, **plot_kwargs):
        self.draw_markers(zero_list, style=style, size=size, **plot_kwargs)

    def plot_branches(self, rmat, **plot_kwargs):
        self.ax.plot(np.real(rmat), np.imag(rmat), **plot_kwargs)

    def label_axes(self, real_point=None, imag_point=None, fontdict=None):
        if real_point is None:
            real_point = self.xmax-0.7j
        if imag_point is None:
            imag_point = 0.3+self.ymax*1j
        self.add_text(real_point, 'Re', fontdict=fontdict)
        self.add_text(imag_point, 'Im', fontdict=fontdict)

    def label_points(self, points, labels, offsets=None, fontdict=None):
        N = len(points)
        if offsets is None:
            offsets = [(0,0)]*N
        elif len(offsets) == 1:
            offsets = N*offsets

        for p, label, offset in zip(points, labels, offsets):
            self.add_text(p, label, xoffset=offset[0], yoffset=offset[1], \
                          fontdict=fontdict)

    def draw_vertical_gridlines(self, xlist=None, \
                                ymin=None, ymax=None, plot_kwargs=None):
        if plot_kwargs is None:
            plot_kwargs = grid_dict

        if ymax is None:
            ymax = self.ymax

        if ymin is None:
            ymin = self.ymin

        if xlist is None:
            xlist = np.arange(0,self.xmax,1)

        for x in xlist:
            self.ax.plot([x,x], [ymin,ymax], **plot_kwargs)

    def draw_horizontal_gridlines(self, ylist=None, \
                                xmin=None, xmax=None, plot_kwargs=None):
        if plot_kwargs is None:
            plot_kwargs = grid_dict

        if xmax is None:
            xmax = self.xmax

        if xmin is None:
            xmin = self.xmin

        if ylist is None:
            ylist = np.arange(self.ymin,self.ymax,1)

        for y in ylist:
            self.ax.plot([int(xmin),int(xmax)], [y,y], **plot_kwargs)

    def draw_xticks(self, ticks=None, dy=0.1, **plot_kwargs):
        kwargs = {'color':'k',\
                  'linewidth':2}
        kwargs.update(plot_kwargs)

        if ticks is None:
            ticks = np.arange(int(self.xmin), int(self.xmax), 1)

        for tick in ticks:
            start_tick = [tick, dy]
            end_tick = [tick, -dy]
            self.ax.plot([tick,tick], [dy,-dy], **kwargs)

    def draw_yticks(self, ticks=None, dx=0.1, **plot_kwargs):
        kwargs = {'color':'k',\
                  'linewidth':2}
        kwargs.update(plot_kwargs)

        if ticks is None:
            ticks = np.arange(int(self.ymin), int(self.ymax)+1, 1)

        for tick in ticks:
            self.ax.plot([-dx,dx], [tick,tick], **kwargs)

    def main_axis(self, ticks=True):
        self.set_arrow_lengths()
        self.axis_off()
        self.draw_axis()
        if ticks:
            self.draw_xticks()
            self.draw_yticks()
        self.label_axes()
        self.ax.set_xlim(self.xlims)
        self.ax.set_ylim(self.ylims)
        plt.tight_layout()

class root_locus_sketch_with_TF(root_locus_sketch):
    def __init__(self, G, ax=None, *args, **kwargs):
        root_locus_sketch.__init__(self, ax, *args, **kwargs)
        self.G = G

    def main(self, kvect=None, markerstyle='bX', linecolor='b',
             plot_branches=True, ticks=True):
        root_locus_sketch.main_axis(self, ticks=ticks)
        self.draw_poles(style=markerstyle)
        if plot_branches:
            self.plot_branches(kvect=kvect,color=linecolor)
        self.draw_zeros()

    def draw_poles(self, style='rX', size=12, **plot_kwargs):
        root_locus_sketch.draw_poles(self, self.G.pole(), style=style, size=size, **plot_kwargs)

    def draw_zeros(self, style='ro', size=12, **plot_kwargs):
        zlist = self.G.zero()
        if len(zlist) > 0:
            root_locus_sketch.draw_zeros(self, zlist, style=style, size=size, **plot_kwargs)

    def plot_branches(self, kvect=None, **plot_kwargs):
        rmat, kout = control.root_locus(self.G, kvect=kvect, Plot=False)
        self.ax.plot(np.real(rmat), np.imag(rmat), **plot_kwargs)

    def label_axes(self, real_point=None, imag_point=None, fontdict=None):
        if real_point is None:
            real_point = self.xmax-0.7j
        if imag_point is None:
            imag_point = 0.3+self.ymax*1j
        self.add_text(real_point, 'Re', fontdict=fontdict)
        self.add_text(imag_point, 'Im', fontdict=fontdict)

    def label_points(self, points, labels, offsets=None, fontdict=None):
        N = len(points)
        if offsets is None:
            offsets = [(0,0)]*N
        elif len(offsets) == 1:
            offsets = N*offsets

        for p, label, offset in zip(points, labels, offsets):
            self.add_text(p, label, xoffset=offset[0], yoffset=offset[1], \
                          fontdict=fontdict)

    def draw_vertical_gridlines(self, xlist=None, \
                                ymin=None, ymax=None, plot_kwargs=None):
        if plot_kwargs is None:
            plot_kwargs = grid_dict

        if ymax is None:
            ymax = self.ymax

        if ymin is None:
            ymin = self.ymin

        if xlist is None:
            xlist = np.arange(0,self.xmax,1)

        for x in xlist:
            self.ax.plot([x,x], [ymin,ymax], **plot_kwargs)

    def draw_horizontal_gridlines(self, ylist=None, \
                                xmin=None, xmax=None, plot_kwargs=None):
        if plot_kwargs is None:
            plot_kwargs = grid_dict

        if xmax is None:
            xmax = self.xmax

        if xmin is None:
            xmin = self.xmin

        if ylist is None:
            ylist = np.arange(self.ymin,self.ymax,1)

        for y in ylist:
            self.ax.plot([int(xmin),int(xmax)], [y,y], **plot_kwargs)

    def draw_xticks(self, ticks=None, dy=0.1, **plot_kwargs):
        kwargs = {'color':'k',\
                  'linewidth':2}
        kwargs.update(plot_kwargs)

        if ticks is None:
            ticks = np.arange(int(self.xmin), int(self.xmax), 1)

        for tick in ticks:
            start_tick = [tick, dy]
            end_tick = [tick, -dy]
            self.ax.plot([tick,tick], [dy,-dy], **kwargs)

    def draw_yticks(self, ticks=None, dx=0.1, **plot_kwargs):
        kwargs = {'color':'k',\
                  'linewidth':2}
        kwargs.update(plot_kwargs)

        if ticks is None:
            ticks = np.arange(int(self.ymin), int(self.ymax), 1)

        for tick in ticks:
            self.ax.plot([-dx,dx], [tick,tick], **kwargs)

class root_locus_sketch_with_TF_2(root_locus_sketch):
    def __init__(self, ax, G, *args, **kwargs):
        root_locus_sketch.__init__(self, ax, *args, **kwargs)
        self.G = G

    def draw_poles(self, style='rX', size=12, zorder=10, **kwargs):
        root_locus_sketch.draw_poles(self, self.G.pole(), style=style, \
                                     size=size, zorder=zorder, **kwargs)

    def draw_zeros(self, style='ro', size=12, zorder=10, **kwargs):
        root_locus_sketch.draw_poles(self, self.G.zero(), style=style, \
                                     size=size, zorder=zorder, \
                                     **kwargs)

    def add_test_point(self, point, style='go', size=12, \
                       label=None, label_offset=0+0j, \
                       **plot_kwargs):
        self.s0 = point
        self.draw_marker(point, style=style, size=size, **plot_kwargs)
        if label is not None:
            self.add_math_text(self.s0+label_offset, label)

    def draw_lines_to_test_point(self, linestyle='k--', **plot_kwargs):
        self.draw_lines_to_point(self.s0, self.G.pole(), \
                                 linestyle=linestyle, **plot_kwargs)
        self.draw_lines_to_point(self.s0, self.G.zero(), \
                                 linestyle=linestyle, **plot_kwargs)

    def build_label_list(self, N, pat):
        labels = []
        for i in range(N):
            j = i+1
            cur_str = pat % j
            labels.append(cur_str)
        return labels

    def make_phi_labels(self):
        N = len(self.G.pole())
        phi_labels = self.build_label_list(N, '\\phi_{%i}')
        return phi_labels

    def make_psi_labels(self):
        N = len(self.G.zero())
        psi_labels = self.build_label_list(N, '\\psi_{%i}')
        return psi_labels

    def label_angles_to_point(self, phi_labels, psi_labels, \
                              phi_offsets=None, psi_offsets=None, \
                              **kwargs):
        if phi_offsets is None:
            np = len(self.G.pole())
            phi_offsets = [0]*np
        if psi_offsets is None:
            nz = len(self.G.zero())
            psi_offsets = [0]*nz

        for p, label, po in zip(self.G.pole(), phi_labels, phi_offsets):
            self.add_math_text(p+po, label, **kwargs)
        for z, label, zo in zip(self.G.zero(), psi_labels, psi_offsets):
            self.add_math_text(z+zo, label, **kwargs)

class root_locus_sketch_two_pole_locations(root_locus_sketch):
    """A class for generating pole locations plots for comparing two
    TFs and their step responses."""
    def __init__(self, G1, G2, label_offsets=[0.2+0.2j, 0.2+0.2j], \
                 markers=['bo','g^'], inds=[1,2], \
                 **kwargs):
        root_locus_sketch.__init__(self, **kwargs)
        self.G1 = G1
        self.G2 = G2
        self.label_offsets = label_offsets
        self.markers = markers
        self.inds = inds

    def draw_and_label_poles(self):
        # get poles and force the imaginary part to be positive
        p1 = self.G1.pole()[0]
        p2 = self.G2.pole()[0]

        for p, m, lo, ind in zip([p1,p2], self.markers, \
                                  self.label_offsets, self.inds):
            p = np.real(p) + 1j*np.abs(np.imag(p))
            self.draw_marker(p, style=m)
            self.draw_marker(np.conj(p), style=m)
            mytext = '$G_{%i}$' % ind
            self.add_text(p, mytext, xoffset=np.real(lo), yoffset=np.imag(lo))

    def main(self, ticks=True):
        self.main_axis(ticks=ticks)
        self.draw_and_label_poles()


#!/usr/bin/python
# -*- coding: utf-8 -*-
from collections import namedtuple
import html
import re
import string
import pandas as pd
import numpy as np
pd.set_option("max_columns", 30)
'''
The tool used is OpenIE 5.0. After following the installation instructions, pass the text file of choice to the system.
If the file contains quotes ' " ', wrap each text within double quotes to avoid the system from raising any exceptions or errors.
Use the simple mode of the system to output extractions that are readable and use the tabbed one to create extractions that can be read by Pandas
'''

'''
The machine readable format of OpenIE 5.0 is similar to named tuples so I am using the same data structure from the collections package. However it does have some differences.
The biggest one being that the text enclosed within the tuples is not within quotes, which throws errors in python.
This is addressed by creating a not so elegant "replace" chain
'''
Context = namedtuple('Context', 'string index')
SimpleArgument = namedtuple('SimpleArgument', 'string index')
TemporalArgument = namedtuple('TemporalArgument', 'string index')
SpatialArgument = namedtuple('SpatialArgument', 'string index')
Relation = namedtuple('Relation', 'string index')
'''
Using the named tuple structure to extract the string enclosed.
Performing a chain of "replace" commands to make the output format compatible with python i.e enclosing string within ""
'''
def col2re(field):
	if field!=0:
		field=field.replace('"','').replace("[","(").replace("{","(").replace("}",")")
		cleanre=re.compile(r"^(.*Argument|Relation|Context)\((.*),List\(([\w\s,\(\)]*)\)\).*$")
		field2=re.sub(cleanre,r'\1("\2",(\3))',field)
		temp=eval(field2)
		return temp.string,temp.index
	else:
		return ""
'''
Splitting the <Argument(s) 2> Column
'''
def col2split(field):
	global argument2
	data=pd.DataFrame('-',range(1),argument2index)
	i=j=k=0
	if field!=0:
		field=field.split("); ")
		if len(field)>1:
			field[:-1]=[field[fi]+")" for fi in range(len(field)-1)]
		for f in field:
			if re.match(r'^SimpleArgument',f):
				i+=1
				if i==1:
					data.loc[0,('SimpleArgument #1','string')]=col2string(f)
					data.loc[0,('SimpleArgument #1','index')]=col2index(f)
				elif i==2:
					data.loc[0,('SimpleArgument #2','string')]=col2string(f)
					data.loc[0,('SimpleArgument #2','index')]=col2index(f)
				elif i==3:
					data.loc[0,('SimpleArgument #3','string')]=col2string(f)
					data.loc[0,('SimpleArgument #3','index')]=col2index(f)
				elif i==4:
					data.loc[0,('SimpleArgument #4','string')]=col2string(f)
					data.loc[0,('SimpleArgument #4','index')]=col2index(f)
			elif re.match(r'^TemporalArgument',f):
				j+=1
				if j==1:
					data.loc[0,('TemporalArgument #1','string')]=col2string(f)
					data.loc[0,('TemporalArgument #1','index')]=col2index(f)
				elif j==2:
					data.loc[0,('TemporalArgument #2','string')]=col2string(f)
					data.loc[0,('TemporalArgument #2','index')]=col2index(f)
				elif j==3:
					data.loc[0,('TemporalArgument #3','string')]=col2string(f)
					data.loc[0,('TemporalArgument #3','index')]=col2index(f)
			elif re.match(r'^SpatialArgument',f):
				k+=1
				if k==1:
					data.loc[0,('SpatialArgument #1','string')]=col2string(f)
					data.loc[0,('SpatialArgument #1','index')]=col2index(f)
				elif k==2:
					data.loc[0,('SpatialArgument #2','string')]=col2string(f)
					data.loc[0,('SpatialArgument #2','index')]=col2index(f)
	argument2=argument2.append(data,ignore_index=True)
'''
Basic raw read from the txt output file of Open IE 5.0
'''
dfcolumns=["Confidence","Context","Argument 1","Relation","Argument(s) 2","Original Text"]
df=pd.read_csv("out_col.txt",sep='\t',names=dfcolumns)
df=df.fillna(0)
'''
Creating a multilevel column header to create the following hierarchy:
Level1:Category	<Confidence> <Context> <Argument 1> <Relation> <Argument(s) 2> <Original Text>
								|			|			|			|___________________________________________________________________________________________________________________________
								|			|			|				|		........		|						|		........		|						|		........		|
Level2:type(Argument 2)		    |  		    |  		    |   	<<SimpleArgument #1>>..<<SimpleArgument #n>> <<TemporalArgument #1>>..<<SimpleArgument #n>> <<SpatialArgument #1>>..<<SpatialArgument #n>>
							____|__		____|__		 ___|___		 ___|___		 		 ___|___	 			 ___|___		 		 ___|___	 			 ___|___		 		 ___|___
							|     |		|     |	  	 |     |         |     |	  	 		 |     |	  	 		 |     |	  	 		 |     |	  	 		 |     |	  	 		 |     |
Level3:datatype			 string index string index string index    string index    		   string index    		   string index    		   string index    		   string index    		   string index
'''
df1columns=[("Original Text","",""),
("Confidence","",""),
("Context","","string"),("Context","","index"),
("Argument 1","","string"),("Argument 1","","index"),
("Relation","","string"),("Relation","","index"),
("Argument(s) 2","SimpleArgument #1","string"),("Argument(s) 2","SimpleArgument #1","index"),
("Argument(s) 2","SimpleArgument #2","string"),("Argument(s) 2","SimpleArgument #2","index"),
("Argument(s) 2","SimpleArgument #3","string"),("Argument(s) 2","SimpleArgument #3","index"),
("Argument(s) 2","SimpleArgument #4","string"),("Argument(s) 2","SimpleArgument #4","index"),
("Argument(s) 2","TemporalArgument #1","string"),("Argument(s) 2","TemporalArgument #1","index"),
("Argument(s) 2","TemporalArgument #2","string"),("Argument(s) 2","TemporalArgument #2","index"),
("Argument(s) 2","TemporalArgument #3","string"),("Argument(s) 2","TemporalArgument #3","index"),
("Argument(s) 2","SpatialArgument #1","string"),("Argument(s) 2","SpatialArgument #1","index"),
("Argument(s) 2","SpatialArgument #2","string"),("Argument(s) 2","SpatialArgument #2","index"),]
df1index=pd.MultiIndex.from_tuples(df1columns,names=["category","type(Argument 2)","datatype"])
df1=pd.DataFrame('-',range(len(df)),df1index)
'''
Creating a multilevel column header to create the following hierarchy:
Level1:type(Argument 2)	<<SimpleArgument #1>>..<<SimpleArgument #n>> <<TemporalArgument #1>>..<<SimpleArgument #n>> <<SpatialArgument #1>>..<<SpatialArgument #n>>
							 ___|___		 		 ___|___	 			 ___|___		 		 ___|___	 			 ___|___		 		 ___|___
						     |     |	  	 		 |     |	  	 		 |     |	  	 		 |     |	  	 		 |     |	  	 		 |     |
Level2:datatype			   string index    		   string index    		   string index    		   string index    		   string index    		   string index
'''
argument2columns=[("SimpleArgument #1","string"),("SimpleArgument #1","index"),
("SimpleArgument #2","string"),("SimpleArgument #2","index"),
("SimpleArgument #3","string"),("SimpleArgument #3","index"),
("SimpleArgument #4","string"),("SimpleArgument #4","index"),
("TemporalArgument #1","string"),("TemporalArgument #1","index"),
("TemporalArgument #2","string"),("TemporalArgument #2","index"),
("TemporalArgument #3","string"),("TemporalArgument #3","index"),
("SpatialArgument #1","string"),("SpatialArgument #1","index"),
("SpatialArgument #2","string"),("SpatialArgument #2","index")]
argument2index=pd.MultiIndex.from_tuples(argument2columns,names=["type(Argument 2)","datatype"])
argument2=pd.DataFrame('-',range(0),argument2index)
'''
Populating the dataframe df1 from the raw data dataframe df by performing transformations and splits
'''
df1.loc[:,'Original Text']=df.loc[:,'Original Text']
df1.loc[:,'Confidence']=df['Confidence'].map(float)
tempre=df.loc[:,'Argument 1'].map(col2re)
df1.loc[:,('Argument 1',"","string")]=tempre.map(lambda x:x[0] if x else np.nan)
df1.loc[:,('Argument 1',"","index")]=tempre.map(lambda x:x[1] if x else np.nan)
tempre=df.loc[:,'Context'].map(col2re)
df1.loc[:,('Context',"","string")]=tempre.map(lambda x:x[0] if x else np.nan)
df1.loc[:,('Context',"","index")]=tempre.map(lambda x:x[1] if x else np.nan)
tempre=df.loc[:,'Relation'].map(col2re)
df1.loc[:,('Relation',"","string")]=tempre.map(lambda x:x[0] if x else np.nan)
df1.loc[:,('Relation',"","index")]=tempre.map(lambda x:x[1] if x else np.nan)
# df['Argument(s) 2'].map(col2split)
# df1['Argument(s) 2']=argument2
df1=df1.apply(html.unescape)
print(df1)
# '''
# Mergin with Claim Id from the Centralifact Database
# '''
# df1=df1.replace('-',np.nan)
# claimsindex=pd.MultiIndex.from_tuples([("Claim Id","",""),("Original Text","","")],names=["category","type(Argument 2)","datatype"])
# claims=pd.read_csv("claimreview_db_spotlight.csv",encoding="utf-8")# can use pd.read_sql_query to get it directly. But working with a specific dataset right now
# claimsdf=pd.DataFrame('-',range(len(claims)),claimsindex)
# claimsdf.loc[:,('Claim Id',"","")]=claims.loc[:,'Claim Id']
# claimsdf.loc[:,('Original Text',"","")]=claims.loc[:,'Original Text']
# claimsdf["Original Text"]=claimsdf["Original Text"].apply(lambda x:'"{}"'.format(x)).apply(html.unescape)
# df1=pd.merge(df1,claimsdf,how='left',on="Original Text")
# '''
# Saving to file
# '''
# df1.to_csv("df1_col.csv")
# df1.to_pickle("df1_col.pkl")


import sqlalchemy
import alembic
import alembic.config

from . import models

class DbEngine:
    def __init__(self):
        self._conn = None
        self._session = None

    def initialize(self):
        alembic_args = ['--raiseerr', 'upgrade', 'head']
        alembic.config.main(argv=alembic_args)

        # TODO: relative (to top level) path... asking for trouble.
        c = alembic.config.Config('./alembic.ini')
        url = c.get_main_option('sqlalchemy.url')

        # Connect to the database
        self._conn = sqlalchemy.create_engine(url, connect_args={'check_same_thread': False})

        # Create a configured "Session" class, then instantiate it
        session_factory = sqlalchemy.orm.sessionmaker(bind=self._conn)
        Session = sqlalchemy.orm.scoped_session(session_factory)
        self._session = Session()

    def deinitialize(self):
        self._session.close()
        self._session = None
        self._conn = None

    def add_or_get_season(self, number):
        s = self._session.query(models.Season) \
                         .filter(models.Season.number == number) \
                         .one_or_none()

        if s is None:
            s = models.Season(number=number)
            self._session.add(s)
            self._session.commit()

        return s

    def get_all_seasons(self):
        return self._session.query(models.Season).all()

    def get_all_season_numbers(self):
        ss = self.get_all_seasons()
        return [s.number for s in ss]

    def get_season(self, number):
        return self._session.query(models.Season) \
                            .filter(models.Season.number == number) \
                            .one_or_none()

    def parse_episode_number(self, number):
        """
        Break up an episode number (string) into the season identifier and
        the episode number (int)
        :param number: the full episode number to parse
        :type number: str
        """
        try:
            season = number[:-2]
            ep = number[-2:]
            episode_number = int(ep)
            return (season, episode_number)
        except (ValueError, TypeError) as e:
            logger.Error('Failed to parse {0} as an episode number.'.format(number))
            return None

    def build_episode_number(self, season, episode_number):
        """
        Put together the season string and an episode number (integer)
        into the full episode id
        :param season: the season id string
        :type season: str
        :param episode_number: the number of the episode within the season
        :type episode_number: int
        """
        episode_string = '{:02d}'.format(episode_number)
        return season + episode_string

    def add_episode(self, number, name):
        """
        :param number: the episode number, e.g., 'K07'
        :type number: str
        :param name: the episode name
        :type name: str, e.g., 'Gamera vs. Zigra'
        """
        (season, episode) = self.parse_episode_number(number)
        s = self.add_or_get_season(season)

        e = models.Episode(number=episode, name=name)
        e.season = s

        self._session.add(e)
        self._session.commit()

        return e

    def add_media_set(self, name):
        m = models.MediaSet(name=name)
        self._session.add(m)
        self._session.commit()

        return m

    def add_to_collection(self, episode, media_set):
        m = models.Media()
        m.episode = episode
        m.media_set = media_set
        self._session.add(m)
        self._session.commit()

    def get_episode_list_by_season(self, season_code):
        s = self.get_season(season_code)
        if s:
            return s.episodes

    def get_all_media_sets(self):
        return self._session.query(models.MediaSet) \
                            .order_by(models.MediaSet.id).all()

    def get_episode_by_code(self, code):
        return self._session.query(models.Episode) \
                            .filter(models.Episode.episode_code == code) \
                            .one_or_none()


from collections import defaultdict
from collections import Counter
import collections
import enum
from re import A
#import numpy as np

import sys
import argparse
import math
import random
from tkinter import N
# https://www.daleseo.com/python-typing/
from typing import Optional
from typing import Union
from typing import List
from typing import Final
from typing import Dict
from typing import Tuple
from typing import Set

import time

# validPalindrome.py : https://github.com/cheoljoo/problemSolving/tree/master/leetcode

timeFlag = 0
debugFlag = 0
import math

class Solution:
    def validPalindrome(self, s: str) -> bool:
        if len(s) <= 2:
            return True
        start = 0
        end = len(s)-1
        while True:
            if start >= end:
                return True
            if s[start] == s[end]:
                start += 1
                end -= 1
            else:
                if start+1 == end:
                    return True
                if self.isPalindrom(s,start+1,end) or self.isPalindrom(s,start,end-1) :
                    return True
                else :
                    return False

    def isPalindrom(self,s:str,start:int,end:int) -> bool:
        while True:
            if start >= end:
                return True
            if s[start] == s[end]:
                start += 1
                end -= 1
            else :
                return False
        return False

def run(s,expect):
    start = time.time()
    A = Solution()
    r = A.validPalindrome(s)
    print(" total_time1 : ", time.time() - start , "-> ", end="")

    if r == expect:
        print("SUCCESS -> ",end="")
    else :
        print("ERROR(",expect,") -> ",sep="",end="")
    print(r, s , end="")
    print()

if (__name__ == "__main__"):
    parser = argparse.ArgumentParser(
        prog='validPalindrome.py',
        description=
        'validPalindrome'
    )
    parser.add_argument( '--debug', '-d' , action='store_const' , const=1 , help='debug on')

    args = parser.parse_args()
    debug = args.debug
    if not debug:
        debug = 0

    print('validPalindrome problem :')

    run("aguokepatgbnvfqmgmlcupuufxoohdfpgjdmysgvhmvffcnqxjjxqncffvmhvgsymdjgpfdhooxfuupuculmgmqfvnbgtapekouga",True)
    run("aba",True)
    run("a",True)
    run("abc",False)
    run("abca",True)
    run("acba",True)
    run("abcdba",True)
    run("acbbdca",True)
    run("acbcad",True)
    run("ad",True)
    run("add",True)
    run("adf",False)
    run("abffcda",False)


# -*- coding: utf-8 -*-
# @Time : 2018/10/24 14:58
# @Author : chenpeng
# @Site : pengguoko@163.com
# @File : wuyou.py
from utils.makesession import create_51session
from bs4 import BeautifulSoup
from utils.mongomodel import WuYou, WuYouId

home = "https://ehire.51job.com"
session = create_51session()

def sto(s):
    d = s.split("_")
    s = "$".join(d)
    return s

def parse_item(payload):
    for key in list(payload.keys()):
        if key != "__VIEWSTATE" and key != "__EVENTTARGET":
            del payload[key]
    payload["send_cycle"] = "1"
    payload["send_time"] = "7"
    payload["send_sum"] = "10"
    payload["ctrlSerach$hidIsRecDisplay"] = "1"
    payload["pagerTopNew$ctl06"] = "10"
    payload["cbxColumns$0"] = "AGE"
    payload["cbxColumns$1"] = "WORKYEAR"
    payload["cbxColumns$2"] = "SEX"
    payload["cbxColumns$3"] = "AREA"
    payload["cbxColumns$4"] = "WORKFUNC"
    payload["cbxColumns$5"] = "TOPDEGREE"
    payload["cbxColumns$6"] = "LASTUPDATE"
    payload["cbxColumns$11"] = "TOPMAJOR"
    payload["cbxColumns$12"] = "TOPSCHOOL"
    payload["hidStrAuthority"] = "0"
    payload["hidDownloadNum"] = "20"
    payload["hidKeywordCookie"] = ""
    payload["ctrlSerach$search_expectsalaryf_input"] = "不限"
    payload["ctrlSerach$search_expectsalaryt_input"] = "不限"
    payload["ctrlSerach$search_wyf_input"] = "不限"
    payload["ctrlSerach$search_wyt_input"] = "不限"
    payload["ctrlSerach$search_df_input"] = "不限"
    payload["ctrlSerach$search_dt_input"] = "不限"
    payload["ctrlSerach$search_cursalaryf_input"] = "不限"
    payload["ctrlSerach$search_cursalaryt_input"] = "不限"
    payload["ctrlSerach$search_age_input"] = "年龄"
    payload["ctrlSerach$search_expjobarea_input"] = "期望工作地"
    payload["ctrlSerach$search_forlang_input"] = "语言"
    payload["ctrlSerach$search_fl_input"] = "不限"
    payload["ctrlSerach$search_fllsabilityll_input"] = "不限"
    payload["ctrlSerach$search_englishlevel_input"] = "英语等级"
    payload["ctrlSerach$search_sex_input"] = "性别"
    payload["ctrlSerach$search_major_input"] = "专业"
    payload["ctrlSerach$search_hukou_input"] = "户口"
    payload["ctrlSerach$search_rsmupdate_input"] = "近1年"
    payload["ctrlSerach$search_jobstatus_input"] = "求职状态"
    payload["pagerTopNew$ctl06"] = "50"
    payload["ctrlSerach$hidIsRecDisplay"] = "1"
    # print(payload)
    r = session.post(
        "https://ehire.51job.com/Candidate/SearchResumeNew.aspx", data=payload
    )
    soup = BeautifulSoup(r.text, "lxml")
    div = soup.find("div", {"class": "Common_list-table"})
    trs = div.find_all("tr")
    for item in trs:
        tds = item.find_all("td")
        if len(tds) == 12:
            res = {}
            unid = WuYouId.objects(uid=tds[1].text)
            if unid:
                w = WuYou.objects.filter(uid=tds[1].text).first()
                if "detail" in w.to_mongo():
                    continue
                else:
                    a = tds[1].find("a")
                    url2 = home + a["href"]
                    try:
                        dd = parse_data(url2)
                        for key in dd.keys():
                            w.key = dd[key]
                        w.save()
                    except Exception as e:
                        print(e)
                        print(url2)
                        pass
            else:
                a = tds[1].find("a")
                url2 = home + a["href"]
                try:
                    dd = parse_data(url2)
                    res.update(dd)
                except Exception as e:
                    print(e)
                    print(url2)
                    pass
                res["uid"] = tds[1].text
                res["age"] = tds[2].text
                res["work_years"] = tds[3].text
                res["gender"] = tds[4].text
                res["address"] = tds[5].text
                res["job_category"] = tds[6].text
                res["edu"] = tds[7].text
                res["updated"] = tds[8].text
                res["major"] = tds[9].text
                res["school"] = tds[10].text
                w = WuYou(**res)
                w.save()
                wid = WuYouId(uid=res["uid"])
                wid.save()
                print("find a new person ", res)
    paydata = {}
    # inputs = soup.find_all('input', {"type": "hidden"})
    inputs = soup.find_all("input")
    for item in inputs:
        try:
            if item["value"]:
                paydata[item["name"]] = item["value"]
            else:
                paydata[item["name"]] = ""
        except:
            pass
    div = soup.find("div", {"class": "Search_page-numble"})
    manya = div.find_all("a")
    current_page = 0
    for item in manya:
        try:
            if item["class"] and item["class"][0] == "active":
                current_page = int(item["title"])
        except:
            pass
        try:
            if current_page:
                if item["title"] == str(current_page + 1):
                    paydata["__EVENTTARGET"] = sto(item["id"])
        except:
            pass
    return paydata, current_page

def parse_data(url1):
    res = {}
    r = session.get(url1)
    soup = BeautifulSoup(r.text, "lxml")
    ta = soup.find("table", {"class": "box1"})
    img = ta.find("img")
    if img:
        if img["src"].startswith("//"):
            res["img"] = "https:" + img["src"]
        else:
            res["img"] = home + img["src"].strip()[img["src"].strip().find("/") :]
    else:
        res["img"] = ""
    try:
        div = soup.find("div", {"id": "divResume"})
        ta = div.find("table", {"class": "column"})
        tables = ta.find_all("table")
        resu = ""
        for table in tables:
            resu = resu + table.text
        res["detail"] = resu
    except Exception as e:
        pass
    print(res)
    return res

def next_page():
    r = session.get("https://ehire.51job.com/Candidate/SearchResumeNew.aspx")
    soup = BeautifulSoup(r.text, "lxml")
    # inputs = soup.find_all('input', {"type": "hidden"})
    inputs = soup.find_all("input")
    paydata = {}
    for item in inputs:
        try:
            if item["value"]:
                paydata[item["name"]] = item["value"]
            else:
                paydata[item["name"]] = ""
        except:
            pass
    div = soup.find("div", {"class": "Search_page-numble"})
    manya = div.find_all("a")
    current_page = 0
    for item in manya:
        try:
            if item["class"] and item["class"][0] == "active":
                current_page = int(item["title"])
        except:
            pass
        try:
            if current_page != 0:
                if item["title"] == str(current_page + 1):
                    paydata["__EVENTTARGET"] = sto(item["id"])
        except:
            pass
    return paydata, current_page

def crawl():
    payload, current_page = next_page()
    while current_page < 101:
        payload1, current_page1 = parse_item(payload)
        payload = payload1
        current_page = int(current_page1)
        print("the parse page is ", current_page)

crawl()


#!/usr/bin/env python
#
# usage: rgb2cmyk.py [-h] [-v] InFile
#
# positional arguments:
#  InFile                Source image file
#
# optional arguments:
#  -h/--help             Show this help message and exit
#  -v/--verbose          Increase output verbosity
#
# Versions:
# - 1.0/2017.11: Initial happy version
#
# TODOs:
# - Support multiple output formats
#
#import argparse
from argparse import ArgumentParser
from argparse import RawTextHelpFormatter
import colortuple
from PIL import Image
from modules.commonlib import warning_msg
from modules.commonlib import error_msg
from modules.commonlib import debug_msg
from modules.commonlib import info_msg
from modules.commonlib import silence_stderr
import os

file_basename = os.path.basename(__file__)
file_version = '1.0|17/11'

def rgb_to_cmyk(r, g, b):
  """
  Convert an RGB color triplet to a CMYK quadruplet.
  """
#  debug_msg( "%d %d %d" % (r, g, b) )
  if r < 3 and g < 3 and b < 3:
    c4 = 0
    m4 = 0
    y4 = 0
    k4 = 100
  else:
    w = max( (r*100)/255, (g*100)/255, (b*100)/255 )
    c4 = ( (w - (r*100)/255)*100 ) / w
    m4 = ( (w - (g*100)/255)*100 ) / w
    y4 = ( (w - (b*100)/255)*100 ) / w
    k4 = 100 - w

  c3 = min(100, (c4*(100 - k4))/100 + k4);
  m3 = min(100, (m4*(100 - k4))/100 + k4);
  y3 = min(100, (y4*(100 - k4))/100 + k4);

  colorC = 255 - (c3*255)/100, 255, 255, 255
  colorM = 255, 255 - (m3*255)/100, 255, 255
  colorY = 255, 255, 255 - (y3*255)/100, 255
  colorK = 255 - (k4*255)/100, 255 - (k4*255)/100, 255 - (k4*255)/100, 255

  return colorC, colorM, colorY, colorK

# Main
def main():

  program_description = "\
 Utility to convert RGB image-files to their CMYK components - v{}\n\
 eg. {} brick-house.png\n\
     {} -h".format(file_version, file_basename, file_basename)

  parser = ArgumentParser(description=program_description, formatter_class=RawTextHelpFormatter)
  parser.add_argument('InFile', type=str, help='input file')
  parser.add_argument('-v', '--verbose', action='store_true', help='increase output verbosity')

  # Read arguments
  args = parser.parse_args()

  # No verbosity by default
  if (not args.verbose):
    silence_stderr()
  else:
    debug_msg("Verbose mode enabled")

  filename_input = args.InFile
  debug_msg( "Input file: '%s'" % (filename_input) )
  format_input = os.path.splitext(filename_input)[1][1:]

  # Main
  debug_msg( "Processing file '%s'..." % (filename_input) )

  img = Image.open(filename_input).convert('RGBA')

  width, height = img.size
  colortuples = img.getcolors()

  imgC = Image.new("RGB", (width, height))
  imgM = Image.new("RGB", (width, height))
  imgY = Image.new("RGB", (width, height))
  imgK = Image.new("RGB", (width, height))

  pix = img.load()
  for x in xrange(0, width):
    for y in xrange(0, height):

      colorC, colorM, colorY, colorK = rgb_to_cmyk(
        pix[x, y][0],
        pix[x, y][1],
        pix[x, y][2]
      )

      imgC.putpixel( (x, y), colorC )
      imgM.putpixel( (x, y), colorM )
      imgY.putpixel( (x, y), colorY )
      imgK.putpixel( (x, y), colorK )

  filename_outputC = filename_input.replace('.' + format_input, '-C.' + format_input)
  debug_msg( "Saving '%s'..." % (filename_outputC) )
  imgC.save( filename_outputC, format_input )

  filename_outputM = filename_input.replace('.' + format_input, '-M.' + format_input)
  debug_msg( "Saving '%s'..." % (filename_outputM) )
  imgM.save( filename_outputM, format_input )

  filename_outputY = filename_input.replace('.' + format_input, '-Y.' + format_input)
  debug_msg( "Saving '%s'..." % (filename_outputY) )
  imgY.save( filename_outputY, format_input )

  filename_outputK = filename_input.replace('.' + format_input, '-K.' + format_input)
  debug_msg( "Saving '%s'..." % (filename_outputK) )
  imgK.save( filename_outputK, format_input )

# Main
if __name__ == '__main__':
  main()


from flask import Flask, jsonify
from transformers import T5ForConditionalGeneration, T5Tokenizer
import json

app = Flask(__name__)

# Load the pre-trained T5 model & tokenizer
question_model = T5ForConditionalGeneration.from_pretrained('priya_t5_squad_v1')
question_tokenizer = T5Tokenizer.from_pretrained('t5-base', model_max_length=1024)

# Defining the get_question_from_answer function
def get_question_from_answer(answer, context):
    text = f"context: {context} answer: {answer} </s>"
    max_len = 256
    encoding = question_tokenizer.encode_plus(
        text,
        max_length=max_len,
        pad_to_max_length=True,
        truncation=True,
        truncation_strategy="longest_first",
        return_tensors="pt"
    )
    input_ids, attention_mask = encoding["input_ids"], encoding["attention_mask"]
    outs = question_model.generate(
        input_ids=input_ids,
        attention_mask=attention_mask,
        early_stopping=True,
        num_beams=5,
        num_return_sequences=1,
        no_repeat_ngram_size=2,
        max_length=200
    )
    dec = [question_tokenizer.decode(ids) for ids in outs]
    question = dec[0].replace("question:", "").strip()
    return question

# Main route for serving questions and answers
@app.route('/')
def index():
    # Reading a JSON file
    file_path = "C:\\Users\\91999\\Downloads\\interviewerSchema.json.json"

    # Read the JSON file
    with open(file_path, "r") as file:
        data = json.load(file)

    # Check if Python, Django, NumPy, and scikit-learn (sklearn) skills are present in the schema
    python_skill_present = any(skill.get("python") for skill in data.get("skills", {}).get("Techskills", []))
    django_skill_present = any(skill.get("django") for skill in data.get("skills", {}).get("Techskills", []))
    numpy_skill_present = any(skill.get("numpy") for skill in data.get("skills", {}).get("Techskills", []))
    sklearn_skill_present = any(skill.get("sklearn") for skill in data.get("skills", {}).get("Techskills", []))

    # Generate questions and answers
    questions_and_answers = []

    # Generate Python questions
    if python_skill_present:
        python_answers = [
            "In Python, the self keyword is used as a convention to refer to the instance of a class within the class itself.",
            "Decorators in python are a way to modify or enhance the behavior of functions or classes without directly modifying their source code.",
            "PEP 8 is a style guide for Python code and PEP 8 specifically focuses on the style conventions for writing Python code.",
            "Python uses private heap space to manage memory.",
            "In Python, inheritance is a mechanism that allows a class to inherit properties (attributes and methods) from another class."
        ]
        for answer in python_answers:
            generated_question = get_question_from_answer(answer, "Python")
            generated_question = generated_question.replace("<pad> ", "").replace("</s>", "")
            questions_and_answers.append({"question": generated_question, "answer": answer})

    # Generate Django questions
    if django_skill_present:
        django_answers = [
            "Django provides built-in support for internationalization (i18n) and localization (l10n), it includes features for translating text, handling date and number formatting based on different locales, and switching between multiple languages in the application",
            "The Django admin site is an automatically generated administration interface that allows authorized users to manage the application's data models.",
            "Django follows a slightly modified version of the MVC architectural pattern, known as Model-View-Template (MVT). Models represent the data structure, views handle the business logic and interaction with models, and templates handle the presentation layer.",
            "Django middleware is a component that sits between the web server and the view function, allowing you to process requests and responses globally.",
            "Django's template system allows developers to separate the presentation logic from the Python code."
        ]
        for answer in django_answers:
            generated_question = get_question_from_answer(answer, "Django")
            generated_question = generated_question.replace("<pad> ", "").replace("</s>", "")
            questions_and_answers.append({"question": generated_question, "answer": answer})

    # Generate NumPy questions
    if numpy_skill_present:
        numpy_answers = [
            "NumPy stands for Numerical Python and it is a powerful library in Python for numerical computations, Its main purpose is to provide efficient and convenient handling of large arrays and matrices of numerical data, along with a collection of mathematical functions to operate on these arrays.",
            "A NumPy array is a grid of values, all of the same data type, and indexed by a tuple of non-negative integers. It is the fundamental data structure in NumPy and provides several advantages over regular Python lists, including faster execution of operations, optimized memory usage, and a wide range of mathematical functions and operations specifically designed for arrays.",
            "A one-dimensional array in NumPy is a simple list of values, while a multi-dimensional array represents a table of elements with rows and columns, similar to a matrix. Multi-dimensional arrays can have any number of dimensions, such as 2D for matrices or 3D for representing volumes of data.",
            "You can reshape or resize a NumPy array using the np.reshape() function or by directly modifying the shape attribute of the array. For example: new_arr = np.reshape(arr, (2, 3)) or arr.shape = (2, 3)",
            "NumPy provides several functions to generate arrays with specific patterns or values, such as np.zeros(), np.ones(), np.arange(), np.linspace(), np.random.rand(), np.random.randint(), and np.eye()."
        ]
        for answer in numpy_answers:
            generated_question = get_question_from_answer(answer, "NumPy")
            generated_question = generated_question.replace("<pad> ", "").replace("</s>", "")
            questions_and_answers.append({"question": generated_question, "answer": answer})

    # Generate scikit-learn (sklearn) questions
    if sklearn_skill_present:
        sklearn_answers = [
            "sklearn provides various modules and components, including datasets for loading and exploring datasets, preprocessing for data preprocessing techniques, model_selection for model selection and evaluation, metrics for performance evaluation metrics, estimators for machine learning algorithms, and ensemble for ensemble methods, among others.",
            "Estimator objects in sklearn are used to fit models to data and make predictions. They encapsulate the learning algorithms and have methods such as fit() to train the model, predict() to make predictions, and score() to evaluate the model's performance.",
            "sklearn provides a datasets module that offers various datasets for practice and experimentation. You can use functions like load_iris(), load_digits(), or fetch_california_housing() to load datasets. Once loaded, you can explore the dataset attributes, such as data, target, feature_names, and target_names.",
            "sklearn offers several preprocessing techniques, including scaling features with StandardScaler or MinMaxScaler, handling missing values with SimpleImputer, encoding categorical variables with OneHotEncoder or LabelEncoder, and reducing dimensionality with techniques like Principal Component Analysis (PCA).",
            "The Pipeline module in scikit-learn allows you to chain multiple preprocessing steps and machine learning models into a single object. It helps in organizing and automating the workflow, enabling easier model building, training, and evaluation."
        ]
        for answer in sklearn_answers:
            generated_question = get_question_from_answer(answer, "scikit-learn")
            generated_question = generated_question.replace("<pad> ", "").replace("</s>", "")
            questions_and_answers.append({"question": generated_question, "answer": answer})

    # If none of the skills are present in the schema
    else:
        return "No Python, Django, NumPy, and scikit-learn tech skill found in the JSON schema."

    # Render the questions and answers as JSON
    return jsonify(questions_and_answers)

# Run the Flask app
if __name__ == '__main__':
    app.run(debug=True)