import atexit
from typing import Callable
import multiprocessing
import sys
import traceback
import gym
from gym.envs.registration import registry as gym_registry
import numpy as np

class Environment:
    """Inherit from this class to test the Swarm on a different problem."""

    action_space = None
    observation_space = None
    reward_range = None
    metadata = None

    def __init__(self, name, n_repeat_action: int=1):
        self._name = name
        self.n_repeat_action = n_repeat_action

    @property
    def unwrapped(self):
        """Completely unwrap this env.

        Returns:
            fractalai.Environment: The base non-wrapped fractalai.Environment instance
        """
        return self

    @property
    def name(self):
        return self._name

    def step(self, action, state=None, n_repeat_action: int=1) -> tuple:
        raise NotImplementedError

    def step_batch(self, actions, states=None, n_repeat_action: int=1) -> tuple:
        raise NotImplementedError

    def reset(self) -> tuple:
        raise NotImplementedError

    def get_state(self):
        raise NotImplementedError

    def set_state(self, state):
        raise NotImplementedError

class AtariEnvironment(Environment):
    """Environment for playing Atari games."""

    def __init__(self, name: str, clone_seeds: bool=True, n_repeat_action: int=1):
        super(AtariEnvironment, self).__init__(name=name, n_repeat_action=n_repeat_action)
        self.clone_seeds = clone_seeds
        spec = gym_registry.spec(name)
        # not actually needed, but we feel safer
        spec.max_episode_steps = None
        spec.max_episode_time = None
        self._env = spec.make()
        self.action_space = self._env.action_space
        self.observation_space = self._env.observation_space
        self.reward_range = self._env.reward_range
        self.metadata = self._env.metadata

    def __getattr__(self, item):
        return getattr(self._env, item)

    @property
    def n_actions(self):
        return self._env.action_space.n

    def get_state(self) -> np.ndarray:
        if self.clone_seeds:
            return self._env.unwrapped.clone_full_state()
        else:
            return self._env.unwrapped.clone_state()

    def set_state(self, state: np.ndarray):
        if self.clone_seeds:
            self._env.unwrapped.restore_full_state(state)
        else:
            self._env.unwrapped.restore_state(state)
        return state

    def step(self, action: np.ndarray, state: np.ndarray = None,
             n_repeat_action: int = None) -> tuple:
        n_repeat_action = n_repeat_action if n_repeat_action is not None else self.n_repeat_action
        if state is not None:
            self.set_state(state)
        reward = 0
        end = False
        info = {"lives": -1}
        for i in range(n_repeat_action):

            obs, _reward, _end, _info = self._env.step(action)
            _info["lives"] = _info.get("ale.lives", -1)
            end = _end or end or info["lives"] > _info["lives"]
            info = _info.copy()
            reward += _reward
            if _end:
                break
        info["terminal"] = _end
        if state is not None:
            new_state = self.get_state()
            return new_state, obs, reward, end, info
        return obs, reward, end, info

    def step_batch(self, actions, states=None, n_repeat_action: [int, np.ndarray]=None) -> tuple:
        """

        :param actions:
        :param states:
        :param n_repeat_action:
        :return:
        """
        n_repeat_action = n_repeat_action if n_repeat_action is not None else self.n_repeat_action
        n_repeat_action = n_repeat_action if isinstance(n_repeat_action, np.ndarray) \
            else np.ones(len(states)) * n_repeat_action
        data = [self.step(action, state, n_repeat_action=dt)
                for action, state, dt in zip(actions, states, n_repeat_action)]
        new_states, observs, rewards, terminals, lives = [], [], [], [], []
        for d in data:
            if states is None:
                obs, _reward, end, info = d
            else:
                new_state, obs, _reward, end, info = d
                new_states.append(new_state)
            observs.append(obs)
            rewards.append(_reward)
            terminals.append(end)
            lives.append(info)
        if states is None:
            return observs, rewards, terminals, lives
        else:
            return new_states, observs, rewards, terminals, lives

    def reset(self, return_state: bool=True):
        if not return_state:
            return self._env.reset()
        else:
            obs = self._env.reset()
            return self.get_state(), obs

    def render(self):
        return self._env.render()

class ESEnvironment(Environment):
    """Environment for Solving Evolutionary Strategies."""

    def __init__(self, name: str, dnn_callable: Callable, n_repeat_action: int=1,
                 max_episode_length=1000, noise_prob: float=0):
        super(ESEnvironment, self).__init__(name=name, n_repeat_action=n_repeat_action)
        self.dnn_callable = dnn_callable
        self._env = gym.make(name)
        self.neural_network = self.dnn_callable()
        self.max_episode_length = max_episode_length
        self.noise_prob = noise_prob

    def __getattr__(self, item):
        return getattr(self._env, item)

    def get_state(self) -> np.ndarray:
        return self.neural_network.get_weights()

    def set_state(self, state: [np.ndarray, list]):
        """
        Sets the microstate of the simulator to the microstate of the target State.
        I will be super grateful if someone shows me how to do this using Open Source code.
        :param state:
        :return:
        """
        self.neural_network.set_weights(state)

    @staticmethod
    def _perturb_weights(weights: [list, np.ndarray],
                         perturbations: [list, np.ndarray]) -> list:
        """
        Updates a set of weights with a gaussian perturbation with sigma equal to self.sigma and
        mean 0.
        :param weights: Set of weights that will be updated.
        :param perturbations: Standard gaussian noise.
        :return: perturbed weights with desired sigma.
        """
        weights_try = []
        for index, noise in enumerate(perturbations):
            weights_try.append(weights[index] + noise)
        return weights_try

    def _normalize_observation(self, obs):
        if "v0" in self.name:
            return obs / 255
        else:
            return obs

    def step(self, action: np.ndarray, state: np.ndarray = None,
             n_repeat_action: int = None) -> tuple:

        n_repeat_action = n_repeat_action if n_repeat_action is not None else self.n_repeat_action

        if state is not None:
            new_weights = self._perturb_weights(state, action)
            self.set_state(new_weights)
        obs = self._env.reset()
        reward = 0
        n_steps = 0
        end = False
        while not end and n_steps < self.max_episode_length:
            if np.random.random() < self.noise_prob:
                nn_action = self._env.action_space.sample()
            else:
                processed_obs = self._normalize_observation(obs.flatten())
                nn_action = self.neural_network.predict(processed_obs)
            for i in range(n_repeat_action):

                obs, _reward, end, info = self._env.step(nn_action)
                reward += _reward
                n_steps += 1

        if state is not None:
            new_state = self.get_state()
            return new_state, obs, reward, False, 0
        return obs, reward, False, 0

    def step_batch(self, actions, states=None, n_repeat_action: int=None) -> tuple:
        n_repeat_action = n_repeat_action if n_repeat_action is not None else self.n_repeat_action
        n_repeat_action = n_repeat_action if isinstance(n_repeat_action, np.ndarray) \
            else np.ones(len(states)) * n_repeat_action
        data = [self.step(action, state, n_repeat_action=dt)
                for action, state, dt in zip(actions, states, n_repeat_action)]
        new_states, observs, rewards, terminals, lives = [], [], [], [], []
        for d in data:
            if states is None:
                obs, _reward, end, info = d
            else:
                new_state, obs, _reward, end, info = d
                new_states.append(new_state)
            observs.append(obs)
            rewards.append(_reward)
            terminals.append(end)
            lives.append(info)
        if states is None:
            return observs, rewards, terminals, lives
        else:
            return new_states, observs, rewards, terminals, lives

    def reset(self, return_state: bool=False):
        if not return_state:
            return self._env.reset()
        else:
            obs = self._env.reset()
            return self.get_state(), obs

def split_similar_chunks(vector: list, n_chunks: int):
    chunk_size = int(np.ceil(len(vector) / n_chunks))
    for i in range(0, len(vector), chunk_size):
        yield vector[i:i + chunk_size]

class ExternalProcess(object):
    """Step environment in a separate process for lock free parallelism.
    It is mostly a copy paste from
    https://github.com/tensorflow/agents/blob/master/agents/tools/wrappers.py
    """

    # Message types for communication via the pipe.
    _ACCESS = 1
    _CALL = 2
    _RESULT = 3
    _EXCEPTION = 4
    _CLOSE = 5

    def __init__(self, constructor):
        """Step environment in a separate process for lock free paralellism.
        The environment will be created in the external process by calling the
        specified callable. This can be an environment class, or a function
        creating the environment and potentially wrapping it. The returned
        environment should not access global variables.
        Args:
          constructor: Callable that creates and returns an OpenAI gym environment.
        Attributes:
          observation_space: The cached observation space of the environment.
          action_space: The cached action space of the environment.
        """
        self._conn, conn = multiprocessing.Pipe()
        self._process = multiprocessing.Process(
            target=self._worker, args=(constructor, conn))
        atexit.register(self.close)
        self._process.start()
        self._observ_space = None
        self._action_space = None

    @property
    def observation_space(self):
        if not self._observ_space:
            self._observ_space = self.__getattr__('observation_space')
        return self._observ_space

    @property
    def action_space(self):
        if not self._action_space:
            self._action_space = self.__getattr__('action_space')
        return self._action_space

    def __getattr__(self, name):
        """Request an attribute from the environment.
        Note that this involves communication with the external process, so it can
        be slow.
        Args:
          name: Attribute to access.
        Returns:
          Value of the attribute.
        """
        self._conn.send((self._ACCESS, name))
        return self._receive()

    def call(self, name, *args, **kwargs):
        """Asynchronously call a method of the external environment.
        Args:
          name: Name of the method to call.
          *args: Positional arguments to forward to the method.
          **kwargs: Keyword arguments to forward to the method.
        Returns:
          Promise object that blocks and provides the return value when called.
        """
        payload = name, args, kwargs
        self._conn.send((self._CALL, payload))
        return self._receive

    def close(self):
        """Send a close message to the external process and join it."""
        try:
            self._conn.send((self._CLOSE, None))
            self._conn.close()
        except IOError:
            # The connection was already closed.
            pass
        self._process.join()

    def set_state(self, state, blocking=True):
        promise = self.call('set_state', state)
        if blocking:
            return promise()
        else:
            return promise

    def step_batch(self, actions, states=None,
                   n_repeat_action: [np.ndarray, int]=None, blocking=True):
        promise = self.call('step_batch', actions, states, n_repeat_action)
        if blocking:
            return promise()
        else:
            return promise

    def step(self, action, state=None, n_repeat_action: int=None, blocking=True):
        """Step the environment.
        Args:
          action: The action to apply to the environment.
          blocking: Whether to wait for the result.
        Returns:
          Transition tuple when blocking, otherwise callable that returns the
          transition tuple.
        """

        promise = self.call('step', action, state, n_repeat_action)
        if blocking:
            return promise()
        else:
            return promise

    def reset(self, blocking=True, return_states: bool=False):
        """Reset the environment.
        Args:
          blocking: Whether to wait for the result.
        Returns:
          New observation when blocking, otherwise callable that returns the new
          observation.
        """
        promise = self.call('reset', return_states=return_states)
        if blocking:
            return promise()
        else:
            return promise

    def _receive(self):
        """Wait for a message from the worker process and return its payload.
        Raises:
          Exception: An exception was raised inside the worker process.
          KeyError: The reveived message is of an unknown type.
        Returns:
          Payload object of the message.
        """
        message, payload = self._conn.recv()
        # Re-raise exceptions in the main process.
        if message == self._EXCEPTION:
            stacktrace = payload
            raise Exception(stacktrace)
        if message == self._RESULT:
            return payload
        raise KeyError('Received message of unexpected type {}'.format(message))

    def _worker(self, constructor, conn):
        """The process waits for actions and sends back environment results.
        Args:
          constructor: Constructor for the OpenAI Gym environment.
          conn: Connection for communication to the main process.
        Raises:
          KeyError: When receiving a message of unknown type.
        """
        try:
            env = constructor()
            env.reset()
            while True:
                try:
                    # Only block for short times to have keyboard exceptions be raised.
                    if not conn.poll(0.1):
                        continue
                    message, payload = conn.recv()
                except (EOFError, KeyboardInterrupt):
                    break
                if message == self._ACCESS:
                    name = payload
                    result = getattr(env, name)
                    conn.send((self._RESULT, result))
                    continue
                if message == self._CALL:
                    name, args, kwargs = payload
                    result = getattr(env, name)(*args, **kwargs)
                    conn.send((self._RESULT, result))
                    continue
                if message == self._CLOSE:
                    assert payload is None
                    break
                raise KeyError('Received message of unknown type {}'.format(message))
        except Exception:  # pylint: disable=broad-except
            import tensorflow as tf
            stacktrace = ''.join(traceback.format_exception(*sys.exc_info()))
            tf.logging.error('Error in environment process: {}'.format(stacktrace))
            conn.send((self._EXCEPTION, stacktrace))
            conn.close()

class BatchEnv(object):
    """Combine multiple environments to step them in batch.
    It is mostly a copy paste from
    https://github.com/tensorflow/agents/blob/master/agents/tools/wrappers.py
    """

    def __init__(self, envs, blocking):
        """Combine multiple environments to step them in batch.
        To step environments in parallel, environments must support a
        `blocking=False` argument to their step and reset functions that makes them
        return callables instead to receive the result at a later time.
        Args:
          envs: List of environments.
          blocking: Step environments after another rather than in parallel.
        Raises:
          ValueError: Environments have different observation or action spaces.
        """
        self._envs = envs
        self._blocking = blocking

    def __len__(self):
        """Number of combined environments."""
        return len(self._envs)

    def __getitem__(self, index):
        """Access an underlying environment by index."""
        return self._envs[index]

    def __getattr__(self, name):
        """Forward unimplemented attributes to one of the original environments.
        Args:
          name: Attribute that was accessed.
        Returns:
          Value behind the attribute name one of the wrapped environments.
        """
        return getattr(self._envs[0], name)

    def _make_transitions(self, actions, states=None, n_repeat_action: [np.ndarray, int]=None):
        states = states if states is not None else [None] * len(actions)
        n_repeat_action = n_repeat_action if isinstance(n_repeat_action, np.ndarray) \
            else np.ones(len(states)) * n_repeat_action
        chunks = len(self._envs)
        states_chunk = split_similar_chunks(states, n_chunks=chunks)
        actions_chunk = split_similar_chunks(actions, n_chunks=chunks)
        repeat_chunk = split_similar_chunks(n_repeat_action, n_chunks=chunks)
        results = []
        for env, states_batch, actions_batch, dt in zip(self._envs,
                                                        states_chunk, actions_chunk, repeat_chunk):
                result = env.step_batch(actions=actions_batch, states=states_batch,
                                        n_repeat_action=dt, blocking=self._blocking)
                results.append(result)

        _states = []
        observs = []
        rewards = []
        terminals = []
        infos = []
        for result in results:
            if self._blocking:
                if states is None:
                    obs, rew, ends, info = result
                else:
                    _sts, obs, rew, ends, info = result
                    _states += _sts
            else:
                if states is None:
                    obs, rew, ends, info = result()
                else:
                    _sts, obs, rew, ends, info = result()
                    _states += _sts
            observs += obs
            rewards += rew
            terminals += ends
            infos += info
        if states is None:
            transitions = observs, rewards, terminals, infos
        else:
            transitions = _states, observs, rewards, terminals, infos
        return transitions

    def step_batch(self, actions, states=None, n_repeat_action: [np.ndarray, int]=None):
        """Forward a batch of actions to the wrapped environments.
        Args:
          actions: Batched action to apply to the environment.
          states: States to be stepped. If None, act on current state.
          n_repeat_action: Number of consecutive times the action will be applied.
        Raises:
          ValueError: Invalid actions.
        Returns:
          Batch of observations, rewards, and done flags.
        """

        if states is None:
            observs, rewards, dones, lives = self._make_transitions(actions, None, n_repeat_action)
        else:
            states, observs, rewards, dones, lives = self._make_transitions(actions, states,
                                                                            n_repeat_action)
        observ = np.stack(observs)
        reward = np.stack(rewards)
        done = np.stack(dones)
        lives = np.stack(lives)
        if states is None:
            return observ, reward, done, lives
        else:
            return states, observs, rewards, dones, lives

    def sync_states(self, state, blocking: bool=True):
        for env in self._envs:
            try:
                env.set_state(state, blocking=blocking)
            except EOFError:
                continue

    def reset(self, indices=None, return_states: bool=True):
        """Reset the environment and convert the resulting observation.
        Args:
          indices: The batch indices of environments to reset; defaults to all.
          return_states: return the corresponding states after reset.
        Returns:
          Batch of observations.
        """
        if indices is None:
            indices = np.arange(len(self._envs))
        if self._blocking:
            observs = [self._envs[index].reset(return_states=return_states) for index in indices]
        else:
            transitions = [self._envs[index].reset(blocking=False,
                                                   return_states=return_states)
                           for index in indices]
            transitions = [trans() for trans in transitions]
            states, observs = zip(*transitions)

        observ = np.stack(observs)
        if return_states:
            return np.array(states), observ
        return observ

    def close(self):
        """Send close messages to the external process and join them."""
        for env in self._envs:
            if hasattr(env, 'close'):
                env.close()

def env_callable(name, env_class, *args, **kwargs):
    def _dummy():
        return env_class(name, *args, **kwargs)
    return _dummy

class ParallelEnvironment(Environment):
    """Wrap any environment to be stepped in parallel."""

    def __init__(self, name, env_class, n_workers: int=8, blocking: bool=True, *args, **kwargs):
        """

        :param name: Name of the Environment
        :param env_class: Class of the environment to be wrapped.
        :param n_workers: number of workers that will be used.
        :param blocking: step the environments asynchronously.
        :param args: args of the environment that will be parallelized.
        :param kwargs: kwargs of the environment that will be parallelized.
        """
        super(ParallelEnvironment, self).__init__(name=name)
        self._env = env_callable(name, env_class, *args, **kwargs)()
        envs = [ExternalProcess(constructor=env_callable(name, env_class, *args, **kwargs))
                for _ in range(n_workers)]
        self._batch_env = BatchEnv(envs, blocking)
        self.action_space = self._env.action_space
        self.observation_space = self._env.observation_space

    def __getattr__(self, item):
        return getattr(self._env, item)

    def step_batch(self, actions: np.ndarray, states: np.ndarray=None,
                   n_repeat_action: [np.ndarray, int]=None):
        return self._batch_env.step_batch(actions=actions, states=states,
                                          n_repeat_action=n_repeat_action)

    def step(self, action: np.ndarray, state: np.ndarray=None, n_repeat_action: int=None):
        return self._env.step(action=action, state=state, n_repeat_action=n_repeat_action)

    def reset(self, return_state: bool = True, blocking: bool=True):
        state, obs = self._env.reset(return_state=True)
        self.sync_states()
        return state, obs if return_state else obs

    def get_state(self):
        return self._env.get_state()

    def set_state(self, state):
        self._env.set_state(state)
        self.sync_states()

    def sync_states(self):
        self._batch_env.sync_states(self.get_state())


#!/usr/bin/python
# -*- coding: UTF-8 -*-
import zlib
import urllib.request
import urllib
from urllib import request
import http.cookiejar

class cookie(request.BaseHandler):
    def http_request(self, req):
        simple_cookie = 'ASP.NET_SessionId=5up1kajxifag0j45nimvjvru'   #这里填自定义cookie值
        if not req.has_header('Cookie'):
            req.add_unredirected_header('Cookie', simple_cookie)  #如果header中cookie为空，则创建cookie对象并把cookie值放进去
        else:
            cookie = req.get_header('Cookie')
            req.add_unredirected_header('Cookie', simple_cookie + '; ' + cookie)  #如果header中已经存在cookie，则把自定义cookie插到已有cookie的末尾
        return req

header={
        'Accept': 'text/html,application/xhtml xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Encoding': 'gzip, deflate',
        'Accept-Language': 'zh-CN,en-US;q=0.8',
        'Connection': 'keep-alive',
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/43.0.235'
    }

postdata =urllib.parse.urlencode({
#"JSESSIONID":"81530CFA37A4AB882173D09544BBEE55",
#"emall-timecard":"{\"userId\":\"00000185\",\"userName\":\"申建利\",\"mobile\":\"18211166905\",\"email\":\"shenjl@i2finance.net\",\"roleId\":\"1\",\"imageUrl\":\"http://210.14.78.246:82/images/3/00000202/1483092273778.jpg\"}"
}).encode('utf-8')
#JSESSIONID=81530CFA37A4AB882173D09544BBEE55; Path=/i2work-timecard
#emall-timecard=}; Path=/

header = {
"Accept":"text/html,application/xhtml xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Encoding":"gzip, deflate",
"Accept-Language":"zh-CN,en-US;q=0.8",
"Connection":"keep-alive",
"Host":"bird2.i2soft.net",
"X-Requested-Width":"com.i2worker",
"User-Agent":"Mozilla/5.0 (Linux; U; Android 2.3.7; en-us; Nexus One Build/FRF91) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1/qianxs",
"Cookie":"JSESSIONID=81530CFA37A4AB882173D09544BBEE55"
}

def getTask(url):

    req = urllib.request.Request(url, postdata, header)
    response = urllib.request.urlopen(req)
    if response.info().get('Content-Encoding', "") == 'gzip':  # e.g www.sina.com.cn
        decompressed_data = zlib.decompress(response.read(), 16 + zlib.MAX_WBITS)
        html = decompressed_data.decode('utf8').splitlines(True)[:10]
    else:
        html = response.read()

    # 自动记住cookie
    cj = http.cookiejar.CookieJar()
    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
    r = opener.open(req)
    result = ""
    if r.info().get('Content-Encoding', "") == 'gzip':  # e.g www.sina.com.cn
        decompressed_data = zlib.decompress(r.read(), 16 + zlib.MAX_WBITS)
        result = decompressed_data.decode('utf8').splitlines(True)[:10]
    else:
        result = response.read()
    print(result)
    print("cookie:",cj)
    return html

def getCookieRequest(url):
    mcj = http.cookiejar.MozillaCookieJar()
    cookiehand = urllib.request.HTTPCookieProcessor(mcj)
    opener = urllib.request.build_opener(cookiehand, cookie())
    print(mcj)
    opener.addheaders = [('User-agent', 'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36')]
    u = opener.open(url)
    result = ""
    if u.info().get('Content-Encoding', "") == 'gzip':  # e.g www.sina.com.cn
        decompressed_data = zlib.decompress(u.read(), 16 + zlib.MAX_WBITS)
        result = decompressed_data.decode('utf8').splitlines(True)[:10]
    else:
        result = u.read()
    print(result)
    print(mcj)

if __name__ == '__main__':
    ss = "{\"userId\":\"00000185\",\"userName\":\"申建利\",\"mobile\":\"18211166905\",\"email\":\"shenjl@i2finance.net\",\"roleId\":\"1\",\"imageUrl\":\"http://210.14.78.246:82/images/3/00000202/1483092273778.jpg\"}"
    print(ss)
    taskUrl = "http://bird2.i2soft.net/i2work-timecard/router.do?method=tc.taskModule.getTaskList"
    #html = getTask(taskUrl)
    #print(html)
    getCookieRequest(taskUrl)


# coding: utf-8

"""
    Memsource REST API

    Welcome to Memsource's API documentation. To view our legacy APIs please [visit our documentation](https://wiki.memsource.com/wiki/Memsource_API) and for more information about our new APIs, [visit our blog](https://www.memsource.com/blog/2017/10/24/introducing-rest-apis-qa-with-the-memsource-api-team/).    If you have any questions, please contact [Memsource Support](<mailto:support@memsource.com>).  # noqa: E501

    OpenAPI spec version: Latest

    Generated by: https://github.com/swagger-api/swagger-codegen.git
"""

import pprint
import re  # noqa: F401

import six

from memsource_cli.models.counts_dto import CountsDto  # noqa: F401,E501

class MatchCountsDto(object):
    """NOTE: This class is auto generated by the swagger code generator program.

    Do not edit the class manually.
    """

    """
    Attributes:
      swagger_types (dict): The key is attribute name
                            and the value is attribute type.
      attribute_map (dict): The key is attribute name
                            and the value is json key in definition.
    """
    swagger_types = {
        'match100': 'CountsDto',
        'match95': 'CountsDto',
        'match85': 'CountsDto',
        'match75': 'CountsDto',
        'match50': 'CountsDto',
        'match0': 'CountsDto'
    }

    attribute_map = {
        'match100': 'match100',
        'match95': 'match95',
        'match85': 'match85',
        'match75': 'match75',
        'match50': 'match50',
        'match0': 'match0'
    }

    def __init__(self, match100=None, match95=None, match85=None, match75=None, match50=None, match0=None):  # noqa: E501
        """MatchCountsDto - a model defined in Swagger"""  # noqa: E501

        self._match100 = None
        self._match95 = None
        self._match85 = None
        self._match75 = None
        self._match50 = None
        self._match0 = None
        self.discriminator = None

        if match100 is not None:
            self.match100 = match100
        if match95 is not None:
            self.match95 = match95
        if match85 is not None:
            self.match85 = match85
        if match75 is not None:
            self.match75 = match75
        if match50 is not None:
            self.match50 = match50
        if match0 is not None:
            self.match0 = match0

    @property
    def match100(self):
        """Gets the match100 of this MatchCountsDto.  # noqa: E501

        :return: The match100 of this MatchCountsDto.  # noqa: E501
        :rtype: CountsDto
        """
        return self._match100

    @match100.setter
    def match100(self, match100):
        """Sets the match100 of this MatchCountsDto.

        :param match100: The match100 of this MatchCountsDto.  # noqa: E501
        :type: CountsDto
        """

        self._match100 = match100

    @property
    def match95(self):
        """Gets the match95 of this MatchCountsDto.  # noqa: E501

        :return: The match95 of this MatchCountsDto.  # noqa: E501
        :rtype: CountsDto
        """
        return self._match95

    @match95.setter
    def match95(self, match95):
        """Sets the match95 of this MatchCountsDto.

        :param match95: The match95 of this MatchCountsDto.  # noqa: E501
        :type: CountsDto
        """

        self._match95 = match95

    @property
    def match85(self):
        """Gets the match85 of this MatchCountsDto.  # noqa: E501

        :return: The match85 of this MatchCountsDto.  # noqa: E501
        :rtype: CountsDto
        """
        return self._match85

    @match85.setter
    def match85(self, match85):
        """Sets the match85 of this MatchCountsDto.

        :param match85: The match85 of this MatchCountsDto.  # noqa: E501
        :type: CountsDto
        """

        self._match85 = match85

    @property
    def match75(self):
        """Gets the match75 of this MatchCountsDto.  # noqa: E501

        :return: The match75 of this MatchCountsDto.  # noqa: E501
        :rtype: CountsDto
        """
        return self._match75

    @match75.setter
    def match75(self, match75):
        """Sets the match75 of this MatchCountsDto.

        :param match75: The match75 of this MatchCountsDto.  # noqa: E501
        :type: CountsDto
        """

        self._match75 = match75

    @property
    def match50(self):
        """Gets the match50 of this MatchCountsDto.  # noqa: E501

        :return: The match50 of this MatchCountsDto.  # noqa: E501
        :rtype: CountsDto
        """
        return self._match50

    @match50.setter
    def match50(self, match50):
        """Sets the match50 of this MatchCountsDto.

        :param match50: The match50 of this MatchCountsDto.  # noqa: E501
        :type: CountsDto
        """

        self._match50 = match50

    @property
    def match0(self):
        """Gets the match0 of this MatchCountsDto.  # noqa: E501

        :return: The match0 of this MatchCountsDto.  # noqa: E501
        :rtype: CountsDto
        """
        return self._match0

    @match0.setter
    def match0(self, match0):
        """Sets the match0 of this MatchCountsDto.

        :param match0: The match0 of this MatchCountsDto.  # noqa: E501
        :type: CountsDto
        """

        self._match0 = match0

    def to_dict(self):
        """Returns the model properties as a dict"""
        result = {}

        for attr, _ in six.iteritems(self.swagger_types):
            value = getattr(self, attr)
            if isinstance(value, list):
                result[attr] = list(map(
                    lambda x: x.to_dict() if hasattr(x, "to_dict") else x,
                    value
                ))
            elif hasattr(value, "to_dict"):
                result[attr] = value.to_dict()
            elif isinstance(value, dict):
                result[attr] = dict(map(
                    lambda item: (item[0], item[1].to_dict())
                    if hasattr(item[1], "to_dict") else item,
                    value.items()
                ))
            else:
                result[attr] = value
        if issubclass(MatchCountsDto, dict):
            for key, value in self.items():
                result[key] = value

        return result

    def to_str(self):
        """Returns the string representation of the model"""
        return pprint.pformat(self.to_dict())

    def __repr__(self):
        """For `print` and `pprint`"""
        return self.to_str()

    def __eq__(self, other):
        """Returns true if both objects are equal"""
        if not isinstance(other, MatchCountsDto):
            return False

        return self.__dict__ == other.__dict__

    def __ne__(self, other):
        """Returns true if both objects are not equal"""
        return not self == other


# -*- coding: utf-8 -*-
"""
Created on Tue May 18 16:50:38 2021

@author: tudor
"""

#Import needed packages
import torch
import torch.nn as nn
from torchvision.datasets import CIFAR10
from torchvision.transforms import transforms
from torch.utils.data import DataLoader
from torch.optim import Adam
from torch.autograd import Variable
import numpy as np
from MyDataset import *

class Unit(nn.Module):
    def __init__(self, in_channels, out_channels):
        super(Unit, self).__init__()

        self.conv = nn.Conv2d(in_channels=in_channels,kernel_size=3,out_channels=out_channels,stride=1,padding=1)
        self.bn = nn.BatchNorm2d(num_features=out_channels)
        self.relu = nn.ReLU()

    def forward(self, input):
        output = self.conv(input)
        output = self.bn(output)
        output = self.relu(output)

        return output

class SimpleNet(nn.Module):
    def __init__(self,num_classes=2):
        super(SimpleNet,self).__init__()

        #Create 14 layers of the unit with max pooling in between
        self.unit1 = Unit(in_channels=3,out_channels=32)
        self.unit2 = Unit(in_channels=32, out_channels=32)
        self.unit3 = Unit(in_channels=32, out_channels=32)

        self.pool1 = nn.MaxPool2d(kernel_size=2)

        self.unit4 = Unit(in_channels=32, out_channels=64)
        self.unit5 = Unit(in_channels=64, out_channels=64)
        self.unit6 = Unit(in_channels=64, out_channels=64)
        self.unit7 = Unit(in_channels=64, out_channels=64)

        self.pool2 = nn.MaxPool2d(kernel_size=2)

        self.unit8 = Unit(in_channels=64, out_channels=128)
        self.unit9 = Unit(in_channels=128, out_channels=128)
        self.unit10 = Unit(in_channels=128, out_channels=128)
        self.unit11 = Unit(in_channels=128, out_channels=128)

        self.pool3 = nn.MaxPool2d(kernel_size=2)

        self.unit12 = Unit(in_channels=128, out_channels=128)
        self.unit13 = Unit(in_channels=128, out_channels=128)
        self.unit14 = Unit(in_channels=128, out_channels=128)

        self.avgpool = nn.AvgPool2d(kernel_size=4)

        #Add all the units into the Sequential layer in exact order
        self.net = nn.Sequential(self.unit1, self.unit2, self.unit3, self.pool1, self.unit4, self.unit5, self.unit6
                                 ,self.unit7, self.pool2, self.unit8, self.unit9, self.unit10, self.unit11, self.pool3,
                                 self.unit12, self.unit13, self.unit14, self.avgpool)

        self.fc = nn.Linear(in_features=128*7*7,out_features=num_classes)

    def forward(self, input):
        output = self.net(input)
        output = output.view(-1, 128*7*7)
        output = self.fc(output)
        return output

class Train:
    def __init__(self):
        self.train_loader = self.TrainLoader()
        self.test_loader = self.TestLoader()
        self.model = SimpleNet(num_classes=2)

        self.cuda_avail = torch.cuda.is_available()

        if self.cuda_avail:
            self.model.cuda()

        self.optimizer = optimizer = Adam(self.model.parameters(), lr = 0.0001, weight_decay=0.0001)
        self.loss_fn = nn.CrossEntropyLoss()

    def adjust_learning_rate(self, epoch):
        lr = 0.001

        if epoch > 180:
            lr = lr / 1000000
        elif epoch > 150:
            lr = lr / 100000
        elif epoch > 120:
            lr = lr / 10000
        elif epoch > 90:
            lr = lr / 1000
        elif epoch > 60:
            lr = lr / 100
        elif epoch > 30:
            lr = lr / 10

        for param_group in self.optimizer.param_groups:
            param_group["lr"] = lr

    def save_models(self, epoch):
        torch.save(self.model.state_dict(), "Model_{}.model".format(epoch))
        print("Checkpoint saved")

    def test(self):
        self.model.eval()
        test_acc = 0.0
        for i, (images, labels) in enumerate(self.test_loader):

            if self.cuda_avail:
                images = Variable(images.cuda())
                labels = Variable(labels.cuda())

            # Predict classes using images from the test set
            outputs = self.model(images)
            _, prediction = torch.max(outputs.data, 1)
            # prediction = prediction.cpu().numpy()

            test_acc += torch.sum(torch.eq(prediction, labels.data))

        # Compute the average acc and loss over all 75 test images
        test_acc = test_acc / 75

        return test_acc

    def TestLoader(self):
        dataset = getTestDataset()

        test_loader = DataLoader(dataset, batch_size=5, shuffle=False, num_workers=4)

        return test_loader

    def TrainLoader(self):
        dataset = getDataset()

        train_loader = DataLoader(dataset, batch_size=5, shuffle=False, num_workers=4)

        return train_loader

    def train(self, num_epochs):
        best_acc = 0.0

        for epoch in range(num_epochs):
            self.model.train()
            train_acc = 0.0
            train_loss = 0.0
            for i, (images, labels) in enumerate(self.train_loader):
                # Move images and labels to gpu if available
                if self.cuda_avail:
                    images = Variable(images.cuda())
                    labels = Variable(labels.cuda())

                # Clear all accumulated gradients
                self.optimizer.zero_grad()
                # Predict classes using images from the test set
                outputs = self.model(images)
                # Compute the loss based on the predictions and actual labels
                loss = self.loss_fn(outputs, labels)
                # Backpropagate the loss
                loss.backward()

                # Adjust parameters according to the computed gradients
                self.optimizer.step()

                train_loss += loss.cpu().data.item() * images.size(0)
                _, prediction = torch.max(outputs.data, 1)

                train_acc += torch.sum(prediction == labels.data)

            # Call the learning rate adjustment function
            self.adjust_learning_rate(epoch)

            # Compute the average acc and loss over all 50000 training images
            train_acc = train_acc / 150
            train_loss = train_loss / 150

            # Evaluate on the test set
            test_acc = self.test()

            # Save the model if the test acc is greater than our current best
            if test_acc > best_acc:
                self.save_models(epoch)
                best_acc = test_acc

            # Print the metrics
            print(
                "Epoch {}, Train Accuracy: {} , TrainLoss: {} , Test Accuracy: {}".format(epoch+1, train_acc, train_loss,
                                                                                test_acc))

    def run(self):
        torch.cuda.empty_cache()
        self.train(20)

if __name__ == "__main__":
    train = Train()
    train.run()


import sys
import datetime
import time
import random
import threading
from queue import Queue

from lib.utils import Logger, Dict
from lib.utils.mysql import DataBase
from . import Parser, Searcher, Requestor

thread_lock = threading.Lock()

def lock(thread_func):
    def wrapper(obj, *args, **kwargs):
        thread_lock.acquire()
        try:
            return thread_func(obj, *args, **kwargs)
        except Exception as e:
            raise e
        finally:
            thread_lock.release()
    return wrapper

class BaseUrlFetcher(threading.Thread):
    def __init__(self, request: Requestor):
        threading.Thread.__init__(self, daemon=True)
        self.request = request

class BaseCrawler(threading.Thread):
    def __init__(self, request: Requestor, parser: Parser, url_fetcher: BaseUrlFetcher):
        threading.Thread.__init__(self, daemon=True)
        self.request = request
        self.parser = parser
        self.url_fetcher = url_fetcher

class TargetedCrawler(BaseCrawler):
    def __init__(self, request: Requestor, parser: Parser, url_fetcher: BaseUrlFetcher, database: DataBase, table: str, logger: Logger, target: dict, queue: Queue, current_info: Dict, proxies=None):
        super().__init__(request, parser, url_fetcher)

        self.database = database
        self.table = table
        self.logger = logger

        self.target = target    # {'alias': '', 'host': ''}
        self.queue = queue
        self.current_info = current_info

        self.proxies = proxies
        self.crawled_urls = set([r['url'] for r in self.database.select(self.table, 'url', site=self.target['alias'])])

    def run(self):
        while self.url_fetcher.is_alive() or self.queue.qsize():
            if not self.queue.empty():
                try:
                    self.current_info.query = self.queue.get()
                    query = self.current_info.query
                    topic = query.get('topic', '')
                    soup, url = self.request(query['url'], no_headers=True)
                    if url['url'] not in self.crawled_urls:
                        data = self.parser(soup, url['netloc'], url['tld'])
                        next_page_url = data.pop('next', None)
                        while next_page_url:
                            try:
                                _soup, _url = self.request(next_page_url)
                                _data = self.parser(_soup, _url['netloc'], _url['tld'])
                                data['text'] += _data['text']
                                next_page_url = _data.pop('next', None)
                            except:
                                break
                        # data = {'publish_time': datetime.datetime.now(), 'source': random_str(4), 'title': random_str(8), 'text': random_str(16)}
                        self.save(**data, url=url['url'], site=self.target['alias'], entry_time=datetime.datetime.now(), topic=topic)
                        self.logger.info(f'parse and save (url:{url["url"]}, topic:{topic}, title:{data["title"]}) successfully')
                        self.crawled_urls.add(url['url'])
                    else:
                        self.logger.info(f'{url["url"]} is in database already')
                    self.current_info.pop('query')
                except:
                    self.logger.error(exc_info=sys.exc_info(), message=f'parse and save (url:{url["url"]}, topic:{topic})')
            else:
                time.sleep(2)

    @lock
    def save(self, **data):
        topic = data.pop('topic')
        last_row_id = self.database.insert(self.table, **data)
        self.database.insert(f'{self.table}_topic', news_id=last_row_id, topic_name=topic)


from flask import Flask, request, jsonify
from rq import Queue
from redis import Redis
import rq_dashboard

from slow_job import slow_job

app = Flask(__name__)

app.config.from_object(rq_dashboard.default_settings)
app.register_blueprint(rq_dashboard.blueprint, url_prefix="/dashboard")

redis_conn = Redis(host='redis', port=6379)
q = Queue(connection=redis_conn)

jobs = {}

@app.route('/job/get/<string:id>', methods=['GET',])
def getJobStatus(id):
    if id == "all":
        out_str = "number of jobs: {jobs_len}
job_ids: {job_ids}
jobs: {jobs}".format( jobs_len=str(len(q)), job_ids=q.job_ids, jobs=q.jobs )
        return out_str

    job = q.fetch_job(id)

    if job == None:
        return "ID not found"
    else:
        return job.get_status()

@app.route('/job/add', methods=['POST',])
def addJob():
    json_in = request.get_json()

    job = q.enqueue(slow_job, json_in["job"], result_ttl="30m", timeout="10m")

    return job.get_id()

if __name__ == '__main__':
    app.run(host="0.0.0.0", port=80, debug=True)


"""
url = http://dgfc.dg.gov.cn/dgwebsite_v2/Vendition/ProjectInfo.aspx?new=1
city : 东莞
CO_INDEX : 9
author: 吕三利
小区数量 : 60    2018/2/24

"""
from backup.crawler_base import Crawler
from lxml import etree
from backup.comm_info import Building, House
import requests, re
from backup.tool import Tool

co_index = 9

class Dongwan(Crawler):
    def __init__(self):
        self.url = 'http://dgfc.dg.gov.cn/dgwebsite_v2/Vendition/ProjectInfo.aspx?new=1'
        self.link_url = 'http://dgfc.dg.gov.cn/dgwebsite_v2/Vendition/'
        self.co_index = 9
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.140 Safari/537.36',
        }
        self.area_list = ['东莞市莞城区',
                          '东莞市东城区',
                          '东莞市万江区',
                          '东莞市南城区',
                          '东莞市石龙镇',
                          '东莞市虎门镇',
                          '东莞市中堂镇',
                          '东莞市望牛墩镇',
                          '东莞市麻涌镇',
                          '东莞市石碣镇',
                          '东莞市高埗镇',
                          '东莞市道滘镇',
                          '东莞市洪梅镇',
                          '东莞市长安镇',
                          '东莞市沙田镇',
                          '东莞市厚街镇',
                          '东莞市松山湖',
                          '东莞市寮步镇',
                          '东莞市大岭山镇',
                          '东莞市大朗镇',
                          '东莞市黄江镇',
                          '东莞市樟木头镇',
                          '东莞市凤岗镇',
                          '东莞市塘厦镇',
                          '东莞市谢岗镇',
                          '东莞市清溪镇',
                          '东莞市常平镇',
                          '东莞市桥头镇',
                          '东莞市横沥镇',
                          '东莞市东坑镇',
                          '东莞市企石镇',
                          '东莞市石排镇',
                          '东莞市茶山镇',
                          '东莞市虎门港',
                          '东莞市生态产业园',
                          ]

    def start_crawler(self):
        town_list = self.get_town_name()
        print(town_list)
        view_dict = Tool.get_view_state(self.url,
                                        view_state='//*[@id="__VIEWSTATE"]/@value',
                                        event_validation='//*[@id="__EVENTVALIDATION"]/@value')
        # print(view_dict)
        # print(town_list)
        all_building_url_list = self.get_all_first_page_url(town_list, view_dict)
        print(all_building_url_list)

        house_url_list = self.get_build_detail(all_building_url_list)

        self.get_house_detail(house_url_list)

    def get_house_detail(self, house_url_list):
        print(house_url_list)
        for i in house_url_list:
            try:
                response = requests.get(i, headers=self.headers)
                html = response.text
                house_html = re.search('id=.roomTable.*?id="remarkDiv"', html, re.S | re.M).group()
                house_info_list = re.findall('", i, re.S | re.M).group(1)
                    house.ho_name = re.search("(.*?)	", i, re.S | re.M).group(1)
                    if 'id' in house.ho_name:
                        house.ho_name = re.search('(.*?)', house.ho_name, re.S | re.M).group(1)
                    house.insert_db()

            except Exception as e:
                print('房号错误，co_index={},url={}'.format(co_index, i), e)
        print('房号放入完成')

    def get_build_detail(self, all_building_url_list):
        house_url_list = []
        for i in all_building_url_list:
            try:
                response = requests.get(i, headers=self.headers)
                html = response.text
                tree = etree.HTML(html)
                bo_develops = tree.xpath('//*[@id="content_1"]/div[3]/text()[2]')[0]  # 开发商
                bu_build_size = tree.xpath('//*[@id="houseTable_1"]/tr[2]/td[6]/a/text()')  # 销售面积
                if bu_build_size:
                    bu_build_size = bu_build_size[0]
                bu_pre_sale = tree.xpath('//*[@id="houseTable_1"]/tr[2]/td[1]/a/text()')  # 预售证书
                if bu_pre_sale:
                    bu_pre_sale = bu_pre_sale[0]
                bu_floor = tree.xpath('//*[@id="houseTable_1"]/tr[2]/td[3]/a/text()')[0]  # 总层数
                bu_all_house = tree.xpath('//*[@id="houseTable_1"]/tr[2]/td[4]/a/text()')[0]  # 总套数
                bu_type = tree.xpath('//*[@id="houseTable_1"]/tr[2]/td[5]/a/text()')[0]  # 房屋用途
                build_html = re.search('houseTable_1.*?当前共有', html, re.S | re.M).group()
                build_detail_html = re.findall('class.*?	.*?	.*?	', build_html, re.S | re.M)
                bu_num = re.findall('项目名称：(.*?)
', html, re.S | re.M)[0].strip()
                url_list = []
                for bu in build_detail_html:
                    try:
                        build = Building(co_index)
                        build.bu_id = re.search("href='roomTable.aspx\?id=(.*?)&", bu, re.S | re.M).group(1)
                        build.bu_address = re.search("_blank.*?_blank'>(.*?)	", bu, re.S | re.M).group(
                            1).strip()
                        build.bo_develops = bo_develops
                        build.bu_build_size = bu_build_size
                        build.bu_pre_sale = bu_pre_sale
                        build.bu_num = bu_num
                        build.bu_floor = bu_floor
                        build.bu_all_house = bu_all_house
                        build.bu_type = bu_type
                        for k in self.area_list:
                            if k in build.bu_address:
                                build.area = k
                                continue
                        build.insert_db()
                        house_url = re.search("(roomTable.aspx\?id=.*?&vc=.*?)'", bu, re.S | re.M).group(1)
                        url_list.append('http://dgfc.dg.gov.cn/dgwebsite_v2/Vendition/' + house_url)
                    except Exception as e:
                        print('楼栋错误，co_index={},url={}'.format(co_index, i), e)
                house_url_list = url_list + house_url_list
            except Exception as e:
                print('楼栋错误，co_index={},url={}'.format(co_index, i), e)
        return house_url_list

    @staticmethod
    def get_all_first_page_url(town_list, view_dict):
        all_building_url_list = []
        for i in town_list:
            try:
                data = {
                    'townName': i,
                    '__EVENTVALIDATION': view_dict['__EVENTVALIDATION'],
                    '__VIEWSTATE': view_dict['__VIEWSTATE'],
                }
                res = requests.post('http://dgfc.dg.gov.cn/dgwebsite_v2/Vendition/ProjectInfo.aspx?new=1', data=data)
                html = etree.HTML(res.content.decode())
                url_list = html.xpath('//*[@id="resultTable"]/tr/td[1]/a/@href')
                complete_url_list = []
                for k in url_list:
                    complete_url_list.append('http://dgfc.dg.gov.cn/dgwebsite_v2/Vendition/' + k)
                all_building_url_list = all_building_url_list + complete_url_list
            except Exception as e:
                print(e)
                continue
        return all_building_url_list

    def get_town_name(self):
        res = requests.get(self.url)
        html = etree.HTML(res.content)
        return html.xpath('//*[@id="townName"]/option/@value')


# Python 3.XX

import requests
import json
from test_data import *
from threading import Thread

def test_areas():
    """
    Function tests get method of https://api.hh.ru/areas.
    """
    # get json with areas
    response = requests.get(api_url+"areas")
    json_list = json.loads(response.text)
    areas_id = [""]  # empty element in list also needed for requests testing

    # takes id's of areas from json and put in the list
    def parse_id():
        def parse_areas(areas):
            for area in areas.get("areas"):
                areas_id.append(area.get("id"))
                parse_areas(area)

        for top_level_area in json_list:
            areas_id.append(top_level_area.get("id"))
            parse_areas(top_level_area)

    def run_in_thread(fn):
        def run(*k, **kw):
            t = Thread(target=fn, args=k, kwargs=kw)
            t.start()
            return t

        return run

    @run_in_thread
    def send_request(l, p):
        for id_area in l:
            response = requests.get(api_url + "areas" + "/{}".format(id_area))
            print("In thread: " + p)
            assert response.status_code in success

    parse_id()

    l_1 = [x for x in areas_id if areas_id.index(x) % 2 is 0]
    l_2 = [x for x in areas_id if areas_id.index(x) % 2 is 1]

    thread1 = send_request(l_1, "1")
    thread2 = send_request(l_2, "2")
    thread1.join()
    thread2.join()

    # negative scenarios
    for el in invalid_list:  # from test_data module
        response = requests.get(api_url+"areas"+"/{}".format(el))
        assert response.status_code in client_error

test_areas()


import json
import os
import shutil
import tempfile
import six
import numpy as np

from chainer.training.extensions import LogReport
from array_summary import DictArraySummary

class BetterLogReport(LogReport):
    """ Subclass LogReport to handle numpy arrays reporting """
    def __call__(self, trainer):
        # accumulate the observations
        keys = self._keys
        observation = trainer.observation
        summary = self._summary

        if keys is None:
            summary.add(observation)
        else:
            summary.add({k: observation[k] for k in keys if k in observation})

        if self._trigger(trainer):
            # output the result
            stats = self._summary.compute_mean()
            stats_cpu = {}
            for name, value in six.iteritems(stats):
                if isinstance(value, np.ndarray):
                    value[np.isnan(value)] = 0.
                    stats_cpu[name] = value.tolist()
                else:
                    if np.isnan(value): value = 0.
                    stats_cpu[name] = float(value)  # copy to CPU

            updater = trainer.updater
            stats_cpu['epoch'] = updater.epoch
            stats_cpu['iteration'] = updater.iteration
            # get total number of examples seen so far
            main_iter = updater._iterators['main']
            stats_cpu['n_examples'] = len(main_iter.dataset)*main_iter.epoch_detail
            stats_cpu['elapsed_time'] = trainer.elapsed_time

            if self._postprocess is not None:
                self._postprocess(stats_cpu)

            self._log.append(stats_cpu)

            # write to the log file
            if self._log_name is not None:
                log_name = self._log_name.format(**stats_cpu)
                fd, path = tempfile.mkstemp(prefix=log_name, dir=trainer.out)
                with os.fdopen(fd, 'w') as f:
                    json.dump(self._log, f, indent=4)

                new_path = os.path.join(trainer.out, log_name)
                shutil.move(path, new_path)

            # reset the summary for the next output
            self._init_summary()

    def _init_summary(self):
        self._summary = DictArraySummary()


import numpy as np
import bpy
from bpy.props import FloatProperty

from mathutils import Vector
from mathutils import Matrix
import pickle
import imp

from . import utils
imp.reload(utils)

#RIGSHAPEPATH = "E:\data\googledrive\lib\model/rig.blend"

#---------------------------------------------------------------------------------------
#リグシェイプ
#---------------------------------------------------------------------------------------
def rigshape_change_scale(self,context):
    props = bpy.context.scene.kiarigtools_props

    utils.mode_p()
    for bone in utils.get_selected_bones():
        bone.custom_shape_scale = props.rigshape_scale/bone.length

def rigshape_revert():
    utils.mode_p()
    #bpy.ops.object.mode_set(mode = 'POSE')
    for bone in utils.get_:
        bone.custom_shape = None

def rigshape_append(filepath):
    #filepath = RIGSHAPEPATH
    current_scene_name = bpy.context.scene.name

    #RigShape_Scn 無ければ作成する
    scene = 'RigShape_Scn'
    if bpy.data.scenes.get(scene) is None:
        bpy.ops.scene.new(type='EMPTY')
        bpy.context.scene.name = scene

    utils.sceneActive(scene)

    #append object from .blend file
    with bpy.data.libraries.load(filepath) as (data_from, data_to):
        data_to.objects = data_from.objects

    #link object to current scene
    for obj in data_to.objects:
        if obj is not None:
            utils.sceneLink(obj)

    utils.sceneActive(current_scene_name)

# make rig shepe size the same. It makes active bone  basis.
def make_the_same_size():
    selected = utils.bone.get_selected_bones()
    act = utils.bone.get_active_bone()

    basesize = act.length * act.custom_shape_scale
    for b in selected:
        b.custom_shape_scale = basesize/b.length
        #print(b.length)

#---------------------------------------------------------------------------------------
#Change rig control value
#---------------------------------------------------------------------------------------
RIGARRAY = ('arm','leg')
PROPARRAY = {
    # 'arm': ('ikfk','stretch'),
    # 'leg': ('ikfk','stretch')
    'arm': ('ikfk','clav','hand','stretch'),
    'leg': ('ikfk','foot','stretch')

}

def rig_change_ctr(self,context):
    amt = bpy.context.object
    props = bpy.context.scene.kiarigtools_props

    for r in RIGARRAY:
        for p in PROPARRAY[r]:
            for lr in ('l' , 'r'):

                ctr = 'ctr.%s.%s' % ( r , lr )

                #if ctr in [o.name for o in bpy.data.objects]:
                if ctr in [b.name for b in amt.pose.bones]:
                    prop ='%s.%s' % (p,lr)
                    prop_val = '%s_%s_%s' % (r,p,lr)
                    #print(ctr , prop , prop_val)
                    exec('amt.pose.bones[\'%s\'][\'%s\'] = props.%s' % ( ctr , prop , prop_val ) ) #amt.pose.bones[ctr.arm.l]['ikfk.l'] = props.arm_ikfk_l'
                    exec('amt.pose.bones[\'%s\'].matrix = amt.pose.bones[\'%s\'].matrix' % (ctr,ctr))  # There is a need to update matrix.

    bpy.context.view_layer.update()

def modify_rig_control_panel( rig , lr , propname , value ):
    amt = bpy.context.object
    props = bpy.context.scene.kiarigtools_props

    ctr = 'ctr.%s.%s' % ( rig , lr )

    if ctr in [b.name for b in amt.pose.bones]:
        prop = '%s.%s' % ( propname , lr )
        prop_val = '%s_%s_%s' % ( rig , propname , lr )
        print( ctr , prop , prop_val )
        exec('props.%s = %f' % ( prop_val ,value ) ) #amt.pose.bones[ctr.arm.l]['ikfk.l'] = props.arm_ikfk_l'
        exec('amt.pose.bones[\'%s\'][\'%s\'] = %f ' % ( ctr , prop , value ) ) #amt.pose.bones[ctr.arm.l]['ikfk.l'] = props.arm_ikfk_l'

def modify_rig_control_panel_key( rig , lr , propname ):
    amt = bpy.context.object
    #props = bpy.context.scene.kiarigtools_props

    ctr = 'ctr.%s.%s' % ( rig , lr )
    prop = '%s.%s' % ( propname , lr )

    bone = amt.pose.bones[ ctr ]
    bone.keyframe_insert(data_path='["%s"]' % prop)

    bpy.context.view_layer.update()

#---------------------------------------------------------------------------------------
#matrix copy paste
#---------------------------------------------------------------------------------------
BONE_MATRIX = []
BONE_MATRIX_DIC = {}

def copy_matrix():
    amt = bpy.context.object
    global BONE_MATRIX_DIC
    BONE_MATRIX_DIC.clear()
    utils.mode_p()

    for bone in utils.get_selected_bones():
        BONE_MATRIX_DIC[bone.name] = Matrix(bone.matrix)
        #m = Matrix(bone.matrix)
        #pos =(m[0][3] , m[1][3] , m[2][3]  )
        #bonematrixarray[bone.name] = [Matrix(bone.matrix) , pos]

def paste_matrix():
    for bone in bpy.context.selected_pose_bones:
        bone.matrix = BONE_MATRIX_DIC[bone.name]

    # for b in BONE_MATRIX_DIC:
    #     print(b)


import os
import boto3
from functools import lru_cache

# Name of the dynamodb table to inspect
database_name = os.environ.get("SECRET_DBNAME", "devops-challenge")
# Key column of the table to match against
lookup_keyname = os.environ.get("LOOKUP_KEYNAME", "code_name")
# Value to match against $lookup_keyname
lookup_keyvalue = os.environ.get("LOOKUP_KEYVALUE", "thedoctor")
# Column of the matched row to retrieve
secret_keyname = os.environ.get("SECRET_KEYNAME", "secret_code")

table = boto3.Session() \
    .resource("dynamodb") \
    .Table(database_name)

@lru_cache()
def get_secret():
    """
    Query a dynamodb table for a row matching our codename. Return the secret value.
    :return: str
    """
    result = table.query(
        KeyConditions={lookup_keyname: {
            'AttributeValueList': [lookup_keyvalue],
            'ComparisonOperator': 'EQ'}},
        AttributesToGet=[secret_keyname],
        Limit=1)

    if result["Items"]:
        item = result["Items"].pop()

        return item[secret_keyname]


import grouping
import load_patient_info
import numpy as np
from tqdm import tqdm
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score, classification_report, confusion_matrix

matching = grouping.matching
matching_keys = matching.keys()
patients_info = load_patient_info.patients_info
#######################################################################################################################
x_train_file_names_positive = []
x_test_file_names_positive = []

for i in range(len(matching_keys)):
    if len(x_train_file_names_positive) < 985:
        x_train_file_names_positive.append(matching_keys[i])
    else:
        x_test_file_names_positive.append((matching_keys[i]))

x_train_file_names_negative = []
for i in range(200):
    for item in x_train_file_names_positive:
        x_train_file_names_negative.append(matching[item][i])
x_test_file_names_negative = []
for i in range(200):
    for item in x_test_file_names_positive:
        x_test_file_names_negative.append(matching[item][i])

test = []
for patient in x_test_file_names_positive:
    test.append(patients_info[patient])
for patient in x_test_file_names_positive:
    for non_patient in matching[patient]:
        test.append(patients_info[non_patient])
test = np.array(test)

#######################################################################################################################

for fold_num in range(5):
    print("start " + str(fold_num + 1) + " fold")
    tmp_validation_names_positive = x_train_file_names_positive[fold_num * 197:(fold_num + 1) * 197]
    tmp_training_names_positive = \
        [item for item in x_train_file_names_positive if item not in tmp_validation_names_positive]

    validation_X = []
    for patient in tmp_validation_names_positive:
        validation_X.append(patients_info[patient])
    for patient in tmp_validation_names_positive:
        for non_patient in matching[patient]:
            validation_X.append(patients_info[non_patient])

    this_fold_test_result = np.zeros(len(test))
    this_fold_validation_result = np.zeros(197*201)

    for j in tqdm(range(200)):
        X = []
        for item in tmp_training_names_positive:
            X.append(patients_info[item])
        for item in tmp_training_names_positive:
            X.append(patients_info[matching[item][j]])
        y = np.concatenate((np.zeros(788) + 1, np.zeros(788)), axis=0)
        X = np.array(X)
        logistic = LogisticRegression()
        logistic.fit(X, y)
        this_fold_test_result += logistic.predict_proba(test)[:, 1]
        # this_fold_test_result += logistic.predict(test)
        this_fold_validation_result += logistic.predict_proba(validation_X)[:, 1]
        # this_fold_validation_result += logistic.predict(validation_X)

    np.savetxt("./result/bagging_logistic_regression/fold_" + str(fold_num+1) + "_test", this_fold_test_result)
    np.savetxt("./result/bagging_logistic_regression/fold_" + str(fold_num+1) + "_validation", this_fold_validation_result)


import os

from flask import Flask, request, jsonify, send_from_directory
from googletrans import Translator
import sys

from werkzeug.utils import secure_filename

app = Flask(__name__, static_folder='')

UPLOAD_FOLDER = ''
ALLOWED_EXTENSIONS = set(['txt', 'py'])
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def allowed_file(filename):
    return '.' in filename and filename.rsplit('.', 1)[1] in ALLOWED_EXTENSIONS

# 1 : Fa to En
# 2 : En to Fa

translator = Translator()

@app.route('/<int:mode>', methods=['POST'])
def translate(mode):
    if 'file' not in request.files:
        out = {'status': 'No file part!'}
        return jsonify(out), 400
    file = request.files['file']

    if file.filename == '':
        out = {'status': 'No selected file!'}
        return jsonify(out), 400
    if file and allowed_file(file.filename):
        filename = secure_filename(file.filename)
        file.save(os.path.join(app.config['UPLOAD_FOLDER'], filename))
        src = filename
        dst = 'dst.txt'
        translated(mode, src, dst)
        out = {'file': '/dst.txt'}
        return jsonify(out), 200
    else:
        out = {'status': 'File type not allowed!'}
        return jsonify(out), 400

@app.route('/<filename>', methods=['GET'])
def get_result(filename):
    return send_from_directory(app.static_folder,
                               filename), 200

def fa2EngAlphabets():
    result = {}
    with open('_alphabetsFa.txt' ,  encoding = 'utf-8') as f:
        for line in f:
            line = line.strip()
            tmp = line.split(' ')
            result[tmp[1]] = tmp[0]
    return result

def fa2EngKeyword():
    result = {}
    with open('_keywords.txt' ,  encoding = 'utf-8') as f:
        for line in f:
            line = line.strip()
            tmp = line.split(' ')
            result[tmp[0]] = tmp[1]
    return result

def eng2FaAlphabets():
    result = {}
    with open('_alphabetsEng.txt' ,  encoding = 'utf-8') as f:
        for line in f:
            line = line.strip()
            tmp = line.split(' ')
            result[tmp[0]] = tmp[1]
    return result

def eng2FaKeyword():
    result = {}
    with open('_keywords.txt' ,  encoding = 'utf-8') as f:
        for line in f:
            line = line.strip()
            tmp = line.split(' ')
            result[tmp[1]] = tmp[0]
    return result

def remove(text):
    newText = []
    for line in text:
        newline = line[0:len(line) - 1]
        newText.append(newline)
    return newText

def translated(mode, src, dst):
    global word
    dict_word = fa2EngKeyword() if mode == 1 else eng2FaKeyword()
    dict_alphabet = fa2EngAlphabets() if mode == 1 else eng2FaAlphabets()
    with open(src, encoding = 'utf-8') as f:
        oldText = f.readlines()
        text = remove(oldText)
        translate = []
        for line in text:
            words = line.split(' ')
            tLine = []
            i = 0
            while i < len(words):
                word = words[i]
                if word == '':
                    tLine.append(' ')
                    i += 1
                elif word[0] == "'":
                    sentence = word[1:] + ' '
                    i += 1
                    word = words[i]
                    while word[-1] != "'":
                        sentence += word + ' '
                        i += 1
                        if i == len(words):
                            word = "'"
                        else:
                            word = words[i]
                    sentence += word[:-1]
                    i += 1
                    sentence = sentence[0:-4]
                    if mode == 1:
                        tLine.append('"' + translator.translate(sentence, src='fa',dest='en').text + '"')
                    else:
                        tLine.append('"' + translator.translate(sentence, dest='fa',src='en').text + '"')

                elif word[0] == '"':
                    sentence = word[1:] + ' '
                    i += 1
                    word = words[i]
                    while word[-1] != '"':
                        sentence += word + ' '
                        i += 1
                        if i == len(words):
                            word = '"'
                        else:
                            word = words[i]
                    sentence += word[:-1]
                    i += 1
                    sentence = sentence[0:-4]
                    if mode == 1:
                        print(word)
                        tLine.append('"' + translator.translate(sentence, src='fa',dest='en').text + '"')
                    else:
                        tLine.append('"' + translator.translate(sentence, dest='fa',src='en').text + '"')

                elif word in dict_word:
                    tLine.append(dict_word[word])
                    i += 1
                elif word in dict_alphabet:
                    tLine.append(word)
                    i += 1
                else:
                    if mode == 1:
                        tLine.append(translator.translate(word, src='fa',dest='en').text)
                    else:
                        tLine.append(translator.translate(word, dest='fa',src='en').text)
                    i += 1
            translate.append(tLine)

    with open(dst, 'w', encoding = 'utf-8') as f:
        for line in translate:
            for word in line:
                if word == ' ':
                    f.write(word)
                else:
                    f.write(word + ' ')
            f.write('\n')

if __name__ == '__main__':
    app.run(debug=True, host='0.0.0.0')


#functionality: return a list of tags given a piece of text

import spacy

nlp = spacy.load("en_core_web_sm")

def get_tags(text):
	tags = []
	doc = nlp(text)

	for token in doc:

		if token.pos_ == "ADJ" or token.pos_ == "VERB" or token.pos_ == "PROPN" or token.pos_ == "NOUN":
			tags.append(token.lemma_)

	tags = list(set(tags))

	return tags


import torch
import jieba
import gensim
from transformers import BertTokenizer, BertForMaskedLM
from scipy.special import softmax
import numpy as np
import traceback
import OpenHowNet

def substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, substitution_words, word_freq_dict, substitution_num):

    MAX = 56065

    loss_scores = []
    freq_scores = []
    sim_scores = []
    hownet_scores = []

    for i in range(len(substitution_words)):
        word = substitution_words[i]
        try:
            freq_scores.append(int(word_freq_dict[word]))
        except:
            freq_scores.append(0)
        sentence_splited = row_line.split('\t')[0].split(' ')
        assert source_word in sentence_splited
        sentence = cut_out(sentence_splited, source_word, 5)
        sub_sentence = sentence.replace(source_word, word)
        loss = sent_loss(model, tokenizer, sub_sentence)
        loss_scores.append(loss)
        try:
            similarity = model_word2vector.similarity(source_word, word)
            sim_scores.append(similarity)
        except:
            sim_scores.append(0)
        try:
            similarity = hownet.calculate_word_similarity(source_word, word)
            hownet_scores.append(similarity)
        except:
            hownet_scores.append(0)

    assert len(loss_scores) == len(freq_scores) == len(sim_scores) == len(hownet_scores)
    loss_scores_sorted = sorted(loss_scores)
    loss_ranks = [loss_scores_sorted.index(x) + 1 for x in loss_scores]
    freq_scores_sorted = sorted(freq_scores)
    freq_ranks = [freq_scores_sorted.index(x) + 1 for x in freq_scores]
    sim_scores_sorted = sorted(sim_scores, reverse=True)
    sim_ranks = [sim_scores_sorted.index(x) + 1 for x in sim_scores]
    hownet_scores_sorted = sorted(hownet_scores, reverse=True)
    hownet_ranks = [hownet_scores_sorted.index(x) + 1 for x in hownet_scores]
    # TODO: rank normalization
    all_ranks = [[substitution_word, loss+freq+sim+hownet] for substitution_word, loss, freq, sim, hownet in zip(substitution_words, loss_ranks, freq_ranks, sim_ranks, hownet_ranks)]
    ss_sorted = sorted(all_ranks, key=lambda x:x[1])
    ss_sorted = [x[0] for x in ss_sorted]
    freq_rank_source = int(word_freq_dict[source_word]) if source_word in word_freq_dict else MAX
    try:
        freq_rank_next = int(word_freq_dict[ss_sorted[1]])
    except:
        freq_rank_next = MAX - 1
    if ss_sorted[0] == source_word and freq_rank_source > freq_rank_next and len(ss_sorted)>=2:
        pre_word = ss_sorted[1]
    else:
        pre_word = ss_sorted[0]

    return pre_word, ss_sorted[:substitution_num:]

def cut_out(sentence_splited, difficult_word, radius):
    d_index = sentence_splited.index(difficult_word)
    start_index = d_index - radius if d_index - radius > 0 else 0
    end_index = d_index + radius if d_index + radius < len(sentence_splited) else len(sentence_splited) - 1
    sentence = ''.join(sentence_splited[start_index:end_index:])
    return sentence

def cross_entropy_word(X, i, pos):
    X = softmax(X, axis=1)
    loss = 0
    loss -= np.log10(X[i, pos])
    return loss

def sent_loss(model, tokenizer, sentence):
    tokenize_input = tokenizer.tokenize(sentence)

    len_sen = len(tokenize_input)

    CLS_TOKEN = '[CLS]'
    SEP_TOKEN = '[SEP]'

    tokenize_input.insert(0, CLS_TOKEN)
    tokenize_input.append(SEP_TOKEN)

    input_ids = tokenizer.convert_tokens_to_ids(tokenize_input)

    sentence_loss = 0

    for i, word in enumerate(tokenize_input):

        if word == CLS_TOKEN or word == SEP_TOKEN:
            continue

        orignial_word = tokenize_input[i]
        tokenize_input[i] = '[MASK]'
        mask_input = torch.tensor([tokenizer.convert_tokens_to_ids(tokenize_input)])
        mask_input = mask_input.to('cuda')
        with torch.no_grad():
            logits = model(mask_input)
        word_loss = cross_entropy_word(logits[0][0].cpu().numpy(), i, input_ids[i])
        sentence_loss += word_loss
        tokenize_input[i] = orignial_word

    return np.exp(sentence_loss/len_sen)

def read_ss_result(res_path):
    res = []
    with open(res_path, 'r', encoding='utf-8') as f_res:
        for line in f_res:
            res.append(line.strip().split(' '))
    return res

def read_dataset(data_path):
    sentences = []
    words = []
    row_lines = []
    with open(data_path, 'r', encoding='utf-8') as reader:
        while True:
            line = reader.readline()
            row_lines.append(line)
            if not line:
                break
            row = line.strip().split('\t')
            sentence, word = row[0], row[1]
            sentences.append(''.join(sentence.split(' ')))
            words.append(word)
    return row_lines, sentences, words

def read_dict(dict_path):
    word_freq_dict = {}
    with open(dict_path, 'r', encoding='utf-8') as f_freq:
        for line in f_freq:
            key, _, value = line.strip().split('\t')
            if key not in word_freq_dict:
                word_freq_dict[key] = value
            elif int(value) < int(word_freq_dict[key]):
                word_freq_dict[key] = value
    return word_freq_dict

def save_result(row_line, pre_word, ss_sorted, path):
    with open(path, 'a', encoding='utf-8') as f_ss_res:
        f_ss_res.write(row_line.strip() + '\n' + pre_word + '\n' + ' '.join(ss_sorted) + '\n')

def main():

    MODEL_CACHE = './model/bert-base-chinese'
    WORD_2_VECTOR_MODEL_DIR = './model/merge_sgns_bigram_char300.txt'

    WORD_FREQ_DICT = './dict/modern_chinese_word_freq.txt'

    EVAL_FILE_PATH = './dataset/annotation_data.csv'
    BERT_RES_PATH = './data/bert_ss_res.csv'
    BERT_NO_AUTOREGRESSIVE_RES_PATH = './data/bert_no_autoregressive_ss_res.csv'
    BERT_WWM_RES_PATH = './data/bert_wwm_ss_res.csv'
    BERT_WWM_EXT_RES_PATH = './data/bert_wwm_ext_ss_res.csv'
    ERNIE_RES_PATH = './data/ernie_ss_res.csv'
    MACBERT_RES_PATH = './data/macbert_base_ss_res.csv'
    ROBERTA_RES_PATH = './data/roberta_wwm_ext_ss_res.csv'
    ELECTRA_RES_PATH = './data/electra_ss_res.csv'
    VECTOR_RES_PATH = './data/vector_ss_res.csv'
    DICT_RES_PATH = './data/dict_ss_res.csv'
    HOWNET_RES_PATH = './data/hownet_ss_res.csv'
    HYBRID_RES_PATH = './data/hybrid_ss_res.csv'

    SUBSTITUTION_NUM = 10

    word_2_vector_model_dir = WORD_2_VECTOR_MODEL_DIR
    model_cache = MODEL_CACHE

    word_freq_dict = WORD_FREQ_DICT

    eval_file_path = EVAL_FILE_PATH

    bert_res_path = BERT_RES_PATH
    bert_no_autoregressive_res_path = BERT_NO_AUTOREGRESSIVE_RES_PATH
    bert_wwm_res_path = BERT_WWM_EXT_RES_PATH
    bert_wwm_ext_res_path = BERT_WWM_EXT_RES_PATH
    ernie_res_path = ERNIE_RES_PATH
    macbert_res_path = MACBERT_RES_PATH
    roberta_res_path = ROBERTA_RES_PATH
    electra_res_path = ELECTRA_RES_PATH
    vector_res_path = VECTOR_RES_PATH
    dict_res_path = DICT_RES_PATH
    hownet_res_path = HOWNET_RES_PATH
    hybrid_res_path = HYBRID_RES_PATH

    substitution_num = SUBSTITUTION_NUM

    print('loading models...')
    tokenizer = BertTokenizer.from_pretrained(model_cache)
    model = BertForMaskedLM.from_pretrained(model_cache)
    # OpenHowNet.download()
    hownet = OpenHowNet.HowNetDict(use_sim=True)
    model.to('cuda')
    model.eval()
    print('loading embeddings...')
    model_word2vector = gensim.models.KeyedVectors.load_word2vec_format(word_2_vector_model_dir, binary=False)
    print('loading files...')
    word_freq_dict = read_dict(word_freq_dict)

    bert_res = read_ss_result(bert_res_path)
    bert_no_autoregressive_res = read_ss_result(bert_no_autoregressive_res_path)
    bert_wwm_res = read_ss_result(bert_wwm_res_path)
    bert_wwm_ext_res = read_ss_result(bert_wwm_ext_res_path)
    ernie_res = read_ss_result(ernie_res_path)
    macbert_res = read_ss_result(macbert_res_path)
    roberta_res = read_ss_result(roberta_res_path)
    electra_res = read_ss_result(electra_res_path)
    vector_res = read_ss_result(vector_res_path)
    dict_res = read_ss_result(dict_res_path)
    hownet_res = read_ss_result(hownet_res_path)
    hybrid_res = read_ss_result(hybrid_res_path)

    row_lines, source_sentences, source_words = read_dataset(eval_file_path)

    for (row_line,
        source_sentence,
        source_word,
        bert_subs,
        bert_no_autoregressive_subs,
        bert_wwm_subs,
        bert_wwm_ext_subs,
        ernie_subs,
        macbert_subs,
        roberta_subs,
        electra_subs,
        vector_subs,
        dict_subs,
        hownet_subs,
        hybrid_subs) in (
        zip(row_lines,
        source_sentences,
        source_words,
        bert_res,
        bert_no_autoregressive_res,
        bert_wwm_res,
        bert_wwm_ext_res,
        ernie_res,
        macbert_res,
        roberta_res,
        electra_res,
        vector_res,
        dict_res,
        hownet_res,
        hybrid_res)
        ):
        # 全部运行可能耗时较长，建议注释部分代码块运行需要的测试
        # It may take a long time to run all the code blocks. We recommend to annotate some code blocks to run the required tests
        if bert_subs[0] != 'NULL':
            bert_pre_word, bert_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, bert_subs, word_freq_dict, substitution_num)
        else:
            bert_pre_word = 'NULL'
            bert_ss_sorted = ['NULL']
        # if bert_no_autoregressive_subs[0] != 'NULL':
        #     bert_no_autoregressive_pre_word, bert_no_autoregressive_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, bert_no_autoregressive_subs, word_freq_dict, substitution_num)
        # else:
        #     bert_no_autoregressive_pre_word = 'NULL'
        #     bert_no_autoregressive_ss_sorted = ['NULL']
        # if bert_wwm_subs[0] != 'NULL':
        #     bert_wwm_pre_word, bert_wwm_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, bert_wwm_subs, word_freq_dict, substitution_num)
        # else:
        #     bert_wwm_pre_word = 'NULL'
        #     bert_wwm_ss_sorted = ['NULL']
        # if bert_wwm_ext_subs[0] != 'NULL':
        #     bert_wwm_ext_pre_word, bert_wwm_ext_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, bert_wwm_ext_subs, word_freq_dict, substitution_num)
        # else:
        #     bert_wwm_ext_pre_word = 'NULL'
        #     bert_wwm_ext_ss_sorted = ['NULL']
        # if ernie_subs[0] != 'NULL':
        #     ernie_pre_word, ernie_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, ernie_subs, word_freq_dict, substitution_num)
        # else:
        #     ernie_pre_word = 'NULL'
        #     ernie_ss_sorted = ['NULL']
        # if roberta_subs[0] != 'NULL':
        #     roberta_pre_word, roberta_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, roberta_subs, word_freq_dict, substitution_num)
        # else:
        #     ernie_pre_word = 'NULL'
        #     ernie_ss_sorted = ['NULL']
        # if macbert_subs[0] != 'NULL':
        #     macbert_pre_word, macbert_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, macbert_subs, word_freq_dict, substitution_num)
        # else:
        #     macbert_pre_word = 'NULL'
        #     macbert_ss_sorted = ['NULL']
        # if electra_subs[0] != 'NULL':
        #     electra_pre_word, electra_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, electra_subs, word_freq_dict, substitution_num)
        # else:
        #     eletra_pre_word = 'NULL'
        #     electra_ss_sorted = ['NULL']
        if vector_subs[0] != 'NULL':
            vector_pre_word, vector_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, vector_subs, word_freq_dict, substitution_num)
        else:
            vector_pre_word = 'NULL'
            vector_ss_sorted = ['NULL']
        if dict_subs[0] != 'NULL':
            dict_pre_word, dict_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, dict_subs, word_freq_dict, substitution_num)
        else:
            dict_pre_word = 'NULL'
            dict_ss_sorted = ['NULL']
        if hownet_subs[0] != 'NULL':
            hownet_pre_word, hownet_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, hownet_subs, word_freq_dict, substitution_num)
        else:
            hownet_pre_word = 'NULL'
            hownet_ss_sorted = ['NULL']
        if hybrid_subs[0] != 'NULL':
            hybrid_pre_word, hybrid_ss_sorted = substitute_ranking(row_line, model_word2vector, model, tokenizer, hownet, source_sentence, source_word, hybrid_subs, word_freq_dict, substitution_num)
        else:
            hybrid_pre_word = 'NULL'
            hybrid_ss_sorted = ['NULL']

        save_result(row_line, bert_pre_word, bert_ss_sorted, './data/bert_sr_res.csv')
        # save_result(row_line, bert_no_autoregressive_pre_word, bert_no_autoregressive_ss_sorted, './data/bert_no_autoregressive_sr_res.csv')
        # save_result(row_line, bert_wwm_pre_word, bert_wwm_ss_sorted, './data/bert_wwm_sr_res.csv')
        # save_result(row_line, bert_wwm_ext_pre_word, bert_wwm_ext_ss_sorted, './data/bert_wwm_ext_sr_res.csv')
        # save_result(row_line, ernie_pre_word, ernie_ss_sorted, './data/ernie_sr_res.csv')
        # save_result(row_line, roberta_pre_word, roberta_ss_sorted, './data/roberta_wwm_ext_sr_res.csv')
        # save_result(row_line, macbert_pre_word, macbert_ss_sorted, './data/macbert_sr_res.csv')
        # save_result(row_line, electra_pre_word, electra_ss_sorted, './data/electra_sr_res.csv')
        save_result(row_line, vector_pre_word, vector_ss_sorted, './data/vector_sr_res.csv')
        save_result(row_line, dict_pre_word, dict_ss_sorted, './data/dict_sr_res.csv')
        save_result(row_line, hownet_pre_word, hownet_ss_sorted, './data/hownet_sr_res.csv')
        save_result(row_line, hybrid_pre_word, hybrid_ss_sorted, './data/hybrid_sr_res.csv')

if __name__ == '__main__':
    main()