import re
import textwrap

from collections import OrderedDict
try:
    from collections.abc import Mapping, Sequence as SequenceABC
except ImportError:
    from collections import Mapping, Sequence as SequenceABC

from glycopeptidepy.structure.sequence import ProteinSequence
from glypy.utils.base import opener

class FastaHeader(Mapping):
    """Hold parsed properties of a FASTA sequence's
    definition line.

    This object supports the :class:`Mapping` interface, and
    keys may be accessed by attribute access notation.

    Attributes
    ----------
    defline : str
        The textual representation of definition line which
        lead to the collection of attributes stored therein.
    """
    def __init__(self, mapping, original=None):
        self._mapping = mapping
        if original is None:
            self.defline = self._make_defline()
        else:
            self.defline = original

    def __getitem__(self, key):
        return self._mapping[key]

    def __iter__(self):
        return iter(self._mapping)

    def items(self):
        return self._mapping.items()

    def keys(self):
        return self._mapping.keys()

    def values(self):
        return self._mapping.values()

    def __len__(self):
        return len(self._mapping)

    def __contains__(self, key):
        return key in self._mapping

    def __getattr__(self, key):
        if key == "_mapping":
            raise AttributeError(key)
        try:
            return self._mapping[key]
        except KeyError:
            raise AttributeError(key)

    def __str__(self):
        return self.defline

    def __repr__(self):
        return "{self.__class__.__name__}({self._mapping})".format(self=self)

    def _make_defline(self):
        raise NotImplementedError()

    def __hash__(self):
        return hash(self.defline)

    def __eq__(self, other):
        return str(self) == str(other)

    def __ne__(self, other):
        return not (self == other)

    def __dir__(self):
        base = set(dir(super(FastaHeader, self)))
        keys = set(self._mapping.keys())
        return list(base | keys)

class UnparsableDeflineError(ValueError):
    """Indicate that a definition line could
    not be parsed by the current parser.
    """
    pass

class DefLineParserBase(object):
    """A base class for definition line parser, providing
    some common machinery in the :meth:`__call__` method.

    This class requires that the subclass provide a :meth:`parse`
    method to convert a string (without a ">" prefix). and return
    an instance of :class:`.OrderedDict`.
    """
    def __call__(self, defline):
        if defline.startswith(">"):
            defline = defline[1:]
        return self._make_header(self.parse(defline), defline)

    def _make_header(self, mapping, defline):
        return FastaHeader(mapping, defline)

    def parse(self, defline):
        raise NotImplementedError()

class CallableDefLineParser(DefLineParserBase):
    """Parse a definition line using an arbitrary
    provided :class:`.Callable`.

    Attributes
    ----------
    callable_obj : Callable
    """
    def __init__(self, callable_obj):
        if not callable(callable_obj):
            raise TypeError("Must provide a callable object.")
        self.callable_obj = callable_obj

    def parse(self, defline):
        result = self.callable_obj(defline)
        if isinstance(result, Mapping):
            return OrderedDict(result.items())
        elif isinstance(result, SequenceABC):
            return OrderedDict(enumerate(result))
        else:
            raise UnparsableDeflineError(
                "Failed to parse {!r} using {!r} (return value: {!r})".format(
                    defline, self, result))

def split_on_whitespace(string):
    return re.split("\s+", string)

space_delim_parser = CallableDefLineParser(split_on_whitespace)

class RegexDefLineParser(DefLineParserBase):
    """Parse a definition line using a provided
    regular expression.

    If :attr:`pattern` contains named groups, this will
    produce a :class:`OrderedDict` where the group names
    are keys, otherwise their index will be used as the
    key.

    Attributes
    ----------
    pattern : re.pattern
    """
    def __init__(self, pattern):
        self.pattern = re.compile(pattern)
        self._is_keyed = "?P<" in pattern

    def parse(self, defline):
        match = self.pattern.search(defline)
        if match:
            if not self._is_keyed:
                return OrderedDict(enumerate(match.groups()))
            else:
                return (match.groupdict())
        else:
            raise UnparsableDeflineError(
                "Failed to parse {!r} using {!r}".format(
                    defline, self))

class DispatchingDefLineParser(DefLineParserBase):
    """Attempt to parse a definition line using one of many
    :class:`DeflineParserBase` instances.

    Attributes
    ----------
    graceful : bool
        If :attr:`graceful`, failure to parse a defline
        with all of :attr:`parsers` results in a :class:`FastaHeader`
        no properties other than *defline*
    header_type : type
        The type to use to contain the result of parsing.
        Defaults to :class:`FastaHeader`.
    parsers : list
        A :class:`list` of :class:`DeflineParserBase` instances
    """
    def __init__(self, parsers, graceful=False, header_type=FastaHeader):
        self.parsers = list(parsers)
        self.graceful = graceful
        self.header_type = header_type

    def _make_header(self, mapping, defline):
        return self.header_type(mapping, defline)

    def parse(self, defline):
        for parser in self.parsers:
            try:
                parts = parser.parse(defline)
                return parts
            except UnparsableDeflineError:
                continue
        else:
            if self.graceful:
                return FastaHeader({}, defline)
            else:
                raise UnparsableDeflineError(
                    "Failed to parse {:r} using {:r}".format(
                        defline, self))

uniprot_regex = r"(?P<db>[a-z^\|]+)\|(?P<accession>[A-Z0-9\-]+)\|(?P<name>\S*)(?:\s(?P<description>.+))?"
uniprot_parser = RegexDefLineParser(uniprot_regex)

partial_uniprot_regex = r"(?P<accession>[A-Z0-9]+)\|(?P<name>\S+)"
partial_uniprot_parser = RegexDefLineParser(partial_uniprot_regex)

default_parser = DispatchingDefLineParser([uniprot_parser, partial_uniprot_parser, space_delim_parser])

class FastaFileParser(object):
    def __init__(self, path, defline_parser=default_parser):
        self.state = "defline"
        self.handle = opener(path)
        self.defline = None
        self.sequence_chunks = []
        self.defline_parser = defline_parser
        self._generator = None

    def process_result(self, d):
        return d

    def __iter__(self):
        return self

    def __next__(self):
        if self._generator is None:
            self._generator = self._parse_lines()
        return next(self._generator)

    def next(self):
        return self.__next__()

    def _parse_lines(self):
        for line in self.handle:
            line = line.strip()
            if self.state == 'defline':
                if line.startswith(">"):
                    self.defline = line[1:]
                    self.state = "sequence"
                else:
                    continue
            else:
                if not re.match(r"^(\s+|>)", line):
                    self.sequence_chunks.append(line)
                else:
                    if self.defline is not None:
                        try:
                            yield self.process_result({
                                "name": self.defline_parser(self.defline),
                                "sequence": ''.join(self.sequence_chunks)
                            })
                        except KeyError as e:
                            print(e)
                            pass
                    self.sequence_chunks = []
                    self.defline = None
                    self.state = 'defline'
                    if line[0] == '>':
                        self.defline = re.sub(r"[\n\r]", "", line[1:])
                        self.state = "sequence"

        if len(self.sequence_chunks) > 0:
            try:
                yield self.process_result(
                    {
                        "name": self.defline_parser(self.defline),
                        "sequence": ''.join(self.sequence_chunks)
                    })
            except Exception as e:
                pass

class ProteinFastaFileParser(FastaFileParser):

    def __init__(self, path, defline_parser=default_parser):
        super(ProteinFastaFileParser, self).__init__(path, defline_parser)

    def process_result(self, d):
        p = ProteinSequence(d['name'], d['sequence'], annotations=dict(d['name']))
        return p

class FastaFileWriter(object):

    def __init__(self, handle):
        self.handle = handle

    def write(self, defline, sequence):
        defline = str(defline)
        if not defline.startswith(">"):
            defline = ">" + defline
        self.handle.write(defline)
        self.handle.write("\n")
        self.handle.write(sequence)
        self.handle.write("\n\n")

    def writelines(self, iterable):
        for defline, seq in iterable:
            self.write(defline, seq)

class ProteinFastaFileWriter(FastaFileWriter):

    def write(self, protein):
        defline = ">%s" % protein.name
        seq = '\n'.join(textwrap.wrap(protein.get_sequence(), 80))
        super(ProteinFastaFileWriter, self).write(defline, seq)

    def writelines(self, iterable):
        for protein in iterable:
            self.write(protein)

def read(f, defline_parser=default_parser, reader_type=ProteinFastaFileParser):
    return reader_type(f, defline_parser=defline_parser)


#!/usr/bin/env python3

#    Copyright 2016 RIFT.IO Inc
#
#    Licensed under the Apache License, Version 2.0 (the "License");
#    you may not use this file except in compliance with the License.
#    You may obtain a copy of the License at
#
#        http://www.apache.org/licenses/LICENSE-2.0
#
#    Unless required by applicable law or agreed to in writing, software
#    distributed under the License is distributed on an "AS IS" BASIS,
#    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#    See the License for the specific language governing permissions and
#    limitations under the License.

"""
#
#

@file test_startstop.py
@author Paul Laidler (Paul.Laidler@riftio.com)
@date 12/17/2015
@brief System test of launchpad start and stop functionality
"""

import pytest

import gi
gi.require_version('RwMcYang', '1.0')
from gi.repository import RwMcYang

@pytest.fixture(scope='module')
def proxy(request, mgmt_session):
    '''fixture which returns a proxy to RwMcYang

    Arguments:
        request       - pytest fixture request
        mgmt_session  - mgmt_session fixture - instance of a rift.auto.session class
    '''
    return mgmt_session.proxy(RwMcYang)

@pytest.mark.depends('launchpad')
@pytest.mark.incremental
class TestLaunchpadStartStop:

    @pytest.mark.feature('mission-control')
    def test_stop_launchpad(self, proxy, mgmt_domain_name):
        '''Invoke stop launchpad RPC

        Asserts:
            Launchpad begins test in state 'started'
            Launchpad finishes test in state 'stopped'
        '''

        proxy.wait_for(
                "/mgmt-domain/domain[name='%s']/launchpad/state" % mgmt_domain_name,
                'started',
                timeout=10,
                fail_on=['crashed'])
        stop_launchpad_input = RwMcYang.StopLaunchpadInput(mgmt_domain=mgmt_domain_name)
        stop_launchpad_output = proxy.rpc(stop_launchpad_input)
        proxy.wait_for(
                "/mgmt-domain/domain[name='%s']/launchpad/state" % mgmt_domain_name,
                'stopped',
                timeout=120,
                fail_on=['crashed'])

    @pytest.mark.feature('mission-control')
    def test_start_launchpad(self, proxy, mgmt_domain_name, launchpad_scraper):
        '''Invoke start launchpad RPC

        Asserts:
            Launchpad begins test in state 'stopped'
            Launchpad finishes test in state 'started'
        '''
        proxy.wait_for(
                "/mgmt-domain/domain[name='%s']/launchpad/state" % mgmt_domain_name,
                'stopped',
                timeout=10,
                fail_on=['crashed'])
        start_launchpad_input = RwMcYang.StartLaunchpadInput(mgmt_domain=mgmt_domain_name)
        start_launchpad_output = proxy.rpc(start_launchpad_input)
        proxy.wait_for(
                "/mgmt-domain/domain[name='%s']/launchpad/state" % mgmt_domain_name,
                'started',
                timeout=400,
                fail_on=['crashed'])
        launchpad_scraper.reset()


import numpy as np
import matplotlib.pyplot as plt

import pandas as pd
from sklearn.datasets import load_boston
boston_dataset = load_boston()

boston = pd.DataFrame(boston_dataset.data, columns=boston_dataset.feature_names)
boston['MEDV'] = boston_dataset.target
X = pd.DataFrame(np.c_[boston['LSTAT'], boston['RM']], columns = ['LSTAT','RM'])
Y = boston['MEDV']

from sklearn.model_selection import train_test_split

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size = 0.2, random_state=5)
print(X_train.shape)
print(X_test.shape)
print(Y_train.shape)
print(Y_test.shape)
from sklearn.linear_model import LinearRegression
from sklearn.metrics import mean_squared_error

lin_model = LinearRegression()
lin_model.fit(X_train, Y_train)
# model evaluation for training set
y_train_predict = lin_model.predict(X_train)
rmse = (np.sqrt(mean_squared_error(Y_train, y_train_predict)))
r2 = r2_score(Y_train, y_train_predict)

print("The model performance for training set")
print("--------------------------------------")
print('RMSE is {}'.format(rmse))
print('R2 score is {}'.format(r2))
print("\n")

# model evaluation for testing set
y_test_predict = lin_model.predict(X_test)
rmse = (np.sqrt(mean_squared_error(Y_test, y_test_predict)))
r2 = r2_score(Y_test, y_test_predict)

print("The model performance for testing set")
print("--------------------------------------")
print('RMSE is {}'.format(rmse))
print('R2 score is {}'.format(r2))


#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""
intensity_normalization.normalize.lsq

fit the

Author: Jacob Reinhold (jacob.reinhold@jhu.edu)

Created on: Feb 23, 2020
"""

import logging
import os

import nibabel as nib
import numpy as np

from intensity_normalization.utilities import io
from intensity_normalization.utilities import mask as mask_util
from .fcm import fcm_normalize, find_tissue_mask

logger = logging.getLogger(__name__)

def lsq_normalize(img_dir, mask_dir=None, output_dir=None, write_to_disk=True):
    """
    normalize intensities of a set of MR images by minimizing the squared distance
    between CSF, GM, and WM means within the set

    Args:
        img_dir (str): directory containing MR images
        mask_dir (str): directory containing masks for MR images
        output_dir (str): directory to save images if you do not want them saved in
            same directory as data_dir
        write_to_disk (bool): write the normalized data to disk or nah

    Returns:
        normalized (np.ndarray): last normalized image from img_dir
    """
    input_files = io.glob_nii(img_dir)
    if output_dir is None:
        out_fns = [None] * len(input_files)
    else:
        out_fns = []
        for fn in input_files:
            _, base, ext = io.split_filename(fn)
            out_fns.append(os.path.join(output_dir, base + '_lsq' + ext))
        if not os.path.exists(output_dir):
            os.mkdir(output_dir)

    mask_files = [None] * len(input_files) if mask_dir is None else io.glob_nii(mask_dir)

    standard_tissue_means = None
    normalized = None
    for i, (img_fn, mask_fn, out_fn) in enumerate(zip(input_files, mask_files, out_fns)):
        _, base, _ = io.split_filename(img_fn)
        logger.info('Transforming image {} to standard scale ({:d}/{:d})'.format(base, i + 1, len(input_files)))
        img = io.open_nii(img_fn)
        mask = io.open_nii(mask_fn) if mask_fn is not None else None
        tissue_mem = mask_util.fcm_class_mask(img, mask)
        if standard_tissue_means is None:
            csf_tissue_mask = find_tissue_mask(img, mask, tissue_type='csf')
            csf_normed_data = fcm_normalize(img, csf_tissue_mask).get_fdata()
            standard_tissue_means = calc_tissue_means(csf_normed_data, tissue_mem)
            del csf_tissue_mask, csf_normed_data
        img_data = img.get_fdata()
        tissue_means = calc_tissue_means(img_data, tissue_mem)
        sf = find_scaling_factor(tissue_means, standard_tissue_means)
        logger.debug('Scaling factor for {}: {:0.3e}'.format(base, sf))
        normalized = nib.Nifti1Image(sf * img_data, img.affine, img.header)
        if write_to_disk:
            io.save_nii(normalized, out_fn, is_nii=True)

    return normalized

def calc_tissue_means(img, tissue_mem):
    def weighted_avg(w, x):
        return (w * x).sum() / w.sum()
    tissue_types = tissue_mem.shape[-1]
    weighted_avgs = [weighted_avg(tissue_mem[..., i], img) for i in range(tissue_types)]
    return np.asarray([weighted_avgs]).T

def find_scaling_factor(tissue_means, standard_tissue_means):
    sf = (tissue_means.T @ standard_tissue_means) / (tissue_means.T @ tissue_means)
    return sf.squeeze()


from dragonfly import (Grammar, AppContext, MappingRule, Dictation, Key, Text, Integer, Mimic)

putty_context = AppContext(executable="putty")
kitty_context = AppContext(executable="kitty")
virtualbox_context = AppContext(executable="virtualbox")
bash_context = AppContext(executable="bash")
sh_context = AppContext(executable="sh")
conemu64_context = AppContext(executable="ConEmu64")
grammar = Grammar("bash", context=(putty_context | sh_context | bash_context | kitty_context | virtualbox_context | conemu64_context))
noSpaceNoCaps = Mimic("\\no-caps-on") + Mimic("\\no-space-on")

rules = MappingRule(
    name = "bash",
    mapping = {
      "watch macros": Text("cd ~/Documents/GitHub/code-by-voice && ./node_modules/.bin/gulp") + Key('enter'),

      "pseudo": Text("sudo "),

      # navigation
      "make der": Text("mkdir ") + noSpaceNoCaps,
      "CD": Text("cd ") + noSpaceNoCaps,
      "push dee": Key("p,u,s,h,d,space"),
      "pop dee": Key("p,o,p,d,enter"),
      "dirs": Key("d,i,r,s,enter"),
      "up directory": Text("cd ..") + Key('enter'),
      "list files": Text("ls -l") + Key("enter"),
      "LS": Text("ls"),
      "LS long": Text("ls -l"),
      "go home": Text("cd ~") + Key("enter"),
      "run": Text("./"),
      "cancel": Key("c-c"),
      "exit": Key("c-d"),

      # particular programs
      "aptitude install" : Text("apt-get install "),
      "pseudo aptitude install" : Text("sudo apt-get install "),
      "them": Text("vim "),
      "W get": Text("wget "),
      "flex": Text("flex "),
      "GCC": Text("gcc "),

      "make": Text("make "),
      "make test": Text("make test") + Key("enter"),
      "make all": Text("make -B") + Key("enter"),
      "windows make": Text("mingw32-make.exe") + Key("enter"),
      "windows make all": Text("mingw32-make.exe -B") + Key("enter"),

      "docker": Text("docker "),
      "docker run": Text("docker run "),

      "NPM": Text("npm "),
      "NPM install": Text("npm install"),

      "ebb": Text("eb "),
      "ebb in it": Text("eb init "),
      "ebb create": Text("eb create "),

      "vagrant upper": Text("vagrant up"),
      "vagrant ssh": Text("vagrant ssh"),
      "vagrant destroy": Text("vagrant destroy"),
      "vagrant reload": Text("vagrant reload"),
      "vagrant reload [dash dash] provision": Text("vagrant reload --provision"),
      "CD slash vagrant": Text("cd /vagrant"),

      # tmux
      "tmux attach": Text("tmux attach") + Key("enter"),
      "tmux attach dash tee": Text("tmux attach -t "),
      "tmux new": Text("tmux new -s "),
      "tmux Ellis": Text("tmux ls") + Key("enter"),
      "tmux": Text("tmux"),

      "term detach": Key("c-b, d"),
      "term prefix": Key("c-b"),
      "term vertical": Key("c-b,percent"),
      "term split": Key("c-b,dquote"),
      "term west": Key("c-b,left"),
      "term east": Key("c-b,right"),
      "term north": Key("c-b,up"),
      "term south": Key("c-b,down"),
      "term create window": Key("c-b,c"),
      "term list windows": Key("c-b,w"),
      "term next window": Key("c-b,n"),
      "term previous window": Key("c-b,p"),
      "term name window": Key("c-b,comma"),
      "term zoom": Key("c-b,z"),
      "term copy": Key("c-b,lbracket"),
      "term pane <n>": Key("c-b,q,%(n)d"),
      "term window <n>": Key("c-b,w,%(n)d"),
      "term reload config": Key("c-b,colon") + Text("source-file ~/.tmux.conf") + Key("enter"),

      # git
      "git": Text("git "),
      "get": Text("git "),
      "status": Text("status"),
      "diff": Text("diff"),
      "commit": Text("commit"),
      "origin": Text("origin "),

      "get add ": Text("git add "),
      "get add [dash dash] patch": Text("git add --patch"),
      "get branch": Text("git branch "),
      "get check out": Text("git checkout "),
      "get check out dash bee": Text("git checkout -b "),
      "get clone": Text("git clone "),
      "get commit": Text("git commit -v") + Key('enter'),
      "get commit amend": Text("git commit -v --amend") + Key('enter'),
      "get diff": Text("git diff") + Key("enter"),
      "get fetch": Text("git fetch "),
      "get log ": Text("git log --decorate") + Key("enter"),
      "get merge": Text("git merge "),
      "get pull": Text("git pull") + Key("enter"),
      "get pull [dash dash] rebase": Text("git pull --rebase") + Key("enter"),
      "get push": Text("git push "),
      "get status": Text("git status -uno") + Key("enter"),
      "get status pipe less": Text("git status | less") + Key('enter'),
      "get short status": Text("git status -sb -uno") + Key("enter"),
      "get stash": Text("git stash "),
      "get stash save": Text("git stash save") + Key('enter'),
      "get stash pop": Text("git stash pop") + Key('enter'),
      "get reset": Text("git reset "),
      "get reset hard": Text("git reset --hard "),
      "get reset hard head": Text("git reset --hard HEAD"),
      "get reset soft": Text("git reset --soft "),

      "gerrit push": Text("git push origin HEAD:refs/for/"),
      "get review": Text("/c/Python27/Scripts/git-review.exe "),
      "get review dash dee": Text("/c/Python27/Scripts/git-review.exe -d "),
      "get review dash dee <n>": Text("/c/Python27/Scripts/git-review.exe -d %(n)d"),

      "get kay": Text("gitk") + Key("enter"),
      },
    extras = [
        Dictation("text"),
        Integer("n", 0, 20000),
      ],
    defaults = {
      "n" : 1
      }
    )

grammar.add_rule(rules)

grammar.load()

def unload():
  global grammar
  if grammar: grammar.unload()
  grammar = None


from keras.utils import np_utils
import glob, keras
from keras.layers import Dense, Activation
from keras.layers import LSTM, SpatialDropout1D, Conv1D, GlobalMaxPooling1D, MaxPooling1D, Flatten
from keras.layers.embeddings import Embedding
from keras.models import Sequential
from Scripts.RNN.Preprocess import NgramVocabulary
from sklearn.model_selection import train_test_split

def MakeDataset(pilepathPS, filepathPStest, ngramsSize):
    with open(pilepathPS, 'r', encoding='utf-8') as f:
        lines = f.readlines()
    with open(filepathPStest, 'r', encoding='utf-8') as f2:
        lines.extend(f2.readlines())
    vectors = []
    classes = []
    vocabulary = NgramVocabulary(ngramsSize)
    for line in lines:
        parts = line.split('\t')
        classes.append(int(parts[0]))
        vectors.append(vocabulary.GetIds(parts[1]))

    maxValue = vocabulary.vocabulary.__len__()
    vectors = keras.preprocessing.sequence.pad_sequences(vectors)
    nb_classes = max(classes)+1
    classes = np_utils.to_categorical(classes, nb_classes)
    return vectors, classes, maxValue, nb_classes

pathPS = 'F:\My_Pro\Python\Jobs2\Data\ProfStandarts'
pathPSTest = 'F:\My_Pro\Python\Jobs2\Data\ProfStandartsTest'
name = 'Специалист по интернет-маркетингу'
#name = 'Администратор БД'
ngramsSize =3

vectors, classes, MAX_VALUE, NB_CLASSES = MakeDataset(pathPS+'\\'+name+'.txt',pathPSTest+'\\'+name+'.txt', ngramsSize)
X_train, X_test, y_train, y_test= train_test_split(vectors,classes,test_size=0.3)

N_HIDDEN = 100
EMBEDDING_VECTOR_LENGTH = 32
BATCH_SIZE = 10
RESHAPED = X_train.shape[1]
model = Sequential()
model.add(Embedding(MAX_VALUE, EMBEDDING_VECTOR_LENGTH))

model.add(SpatialDropout1D(0.2))
model.add(LSTM(50, dropout=0.3, recurrent_dropout=0.3, return_sequences=True))
model.add(Conv1D(filters=512, kernel_size=9, activation='relu'))
model.add(GlobalMaxPooling1D())

model.add(Dense(NB_CLASSES, activation="sigmoid"))
model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])

model.fit(X_train, y_train, validation_data=(X_test, y_test), epochs=2, batch_size=16)
scores = model.evaluate(X_test, y_test, verbose=0)
print("Accuracy %.2f%%" % (scores[1] * 100))


'''
Files to modify :
1. common_config : add our custom_dataset
2. MyPath : add path to dataset
3. create  new config files
4. create new datalaoder

'''

#creating a custom dataloader
"""
This code is based on the Torchvision repository, which was licensed under the BSD 3-Clause.
"""
import os
import pickle
import sys
import numpy as np
import torch
from PIL import Image
from torch.utils.data import Dataset
from torchvision.datasets.utils import check_integrity, download_and_extract_archive
import pandas as pd

class EuroSat(Dataset):

    def __init__(self, root, train=True, transform=None):

        super(EuroSat, self).__init__()

        self.root = root

        self.transform = transform

        self.train = train

        self.classes = ['Forest','SeaLake','PermanentCrop','Industrial','River','AnnualCrop','HerbaceousVegetation','Residential','Highway','Pasture']

        self.df = EuroSat.get_data_dict(self.root)

        #keep 75 for training and

        if train:
          self.df = self.df[self.df.train == 1.0].reset_index(drop = True)
        else:
          self.df = self.df[self.df.train == 0.0].reset_index(drop = True)

    @staticmethod
    def get_data_dict(data_dir):

        data_dict = {'path': [], 'label': []}

        for (_, labels, _) in os.walk(data_dir):
            for label in labels:
                label_dir = data_dir + label + '/'
                for (_, _, files) in os.walk(label_dir):
                    for file in files:
                        data_dict['path'].append(label_dir + file)
                        data_dict['label'].append(label)

        df =  pd.DataFrame(data_dict)
        #shuffling the dataframe.
        df = df.sample(frac=1).reset_index(drop=True)

        n = len(df)
        #set 75 for training and   and 25 for testing

        df['train'] = np.zeros(n)

        df.loc[0:int(n * 0.75),'train'] = 1

        return df

    def __getitem__(self, index):
        """
        Args:
            index (int): Index
        Returns:
            dict: {'image': image, 'target': index of target class, 'meta': dict}
        """
        img = self.get_image(index)
        img_size = img.size

        class_name = self.df['label'][index]
        target = self.classes.index(class_name)

        if self.transform is not None:
            img = self.transform(img)

        out = {'image': img, 'target': target, 'meta': {'im_size': img_size, 'index': index, 'class_name': class_name}}

        return out

    def get_image(self, index):
        img_path = self.df['path'][index]
        img = Image.open(img_path)
        return img

    #functions used for plotting images
    def show_img(self,index):
        img =  self.get_image(index)
        plt.imshow(img)

    # will create a batch x batch images
    def plot_batch(self,batch = 8):
        n_images = len(self.df)

        indices = np.random.choice(range(n_images) , size= batch **2)

        #creating an image array
        labels = self.df['label'][indices].reset_index(drop = True)
        images = [np.array(self.get_image(i)) for i in indices]

        fig,axis = plt.subplots(batch,batch,figsize = (20,20))

        for i,ax in enumerate(axis.flatten()):

            ax.imshow(images[i])
            ax.set_title(labels[i])
            ax.xaxis.set_visible(False)
            ax.yaxis.set_visible(False)

    def __len__(self):
        return len(self.df)


# -*- coding: UTF-8 -*-
from django.conf.urls import url
from impresario.views import PersonList2, PersonAdd, PersonEdit, PersonList2Json,CustomerList, CustomerAdd, CustomerEdit

app_name = 'impresario'

urlpatterns = [
    # exactly: /impresario/person/
    url(r'^contact/$', PersonList2.as_view(), name='persons_list'),

    url(r'^persons_data/$', PersonList2Json.as_view(), name="persons_list_json"),

    # exactly: /impresario/person/add
    url(r'^contact/add/$', PersonAdd.as_view(), name='persons_add'),

    # example: /impresario/person/1
    url(r'^contact/(?P<pk>\d+)/$', PersonEdit.as_view(), name='persons_edit'),

    # example: /impresario/
    url(r'^$', CustomerList.as_view(), name='customer_list'),

    # exactly: /impresario/add
    url(r'^add/$', CustomerAdd.as_view(), name='customer_add'),

    # example: /impresario/1
    url(r'^(?P<pk>\d+)/$', CustomerEdit.as_view(), name='customer_edit'),

]


# -*- coding:utf-8 -*-
# author: xiaojie
# datetime: 2020/5/12 10:57
# software: PyCharm

from bert_classification_utils import init_logger, DataProcess
from trainer import Trainer
import torch
from tqdm import tqdm
import logging
import random
from argparse import Namespace
from torch.utils.data import TensorDataset
import numpy as np
import json
import os
import codecs
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
init_logger()

logger = logging.getLogger(__name__)

class BertClassificationTrainModelHandler(DataProcess):

    def __init__(self, config_params):

        config_params.update(config_params.get("hyper_param", {}))
        self.config = Namespace(**config_params)
        self.config.no_cuda = False
        # self.config.model_name_or_path = "E:\\nlp_tools\\bert_models\\bert-base-chinese"
        self.config.model_name_or_path = "/home/hemei/xjie/bert_models/bert-base-chinese"

        self.ADDITIONAL_SPECIAL_TOKENS = self.config.ADDITIONAL_SPECIAL_TOKENS
        self.ADDITIONAL_SPECIAL_TOKENS = ["<e1>", "</e1>", "<e2>", "</e2>"]
        super(BertClassificationTrainModelHandler, self).__init__(self.config)
        # token
        self.tokenizer = self.load_tokenizer(self.config)
        ##########

        ##########
        self.model_save_path = self.config.model_dir

    def data_process(self):
        # 获取正负样本

        data, labels, max_seq_len, other_size = self.get_data_1()

        label_id = {l: ind for ind, l in enumerate(labels)}
        id_label = {v: k for k, v in label_id.items()}
        self.config.max_seq_len = max_seq_len
        self.config.labels = labels
        self.config.label_id = label_id
        self.config.id_label = id_label
        self.config.other_size = other_size

        # data = data[:100]

        random.shuffle(data)
        train_len = int(len(data) * 0.8)
        train_data_ = data[:train_len]
        test_data_ = data[train_len:]

        self.train_data = self._get_data(train_data_, label_id, "train")

        self.test_data = self._get_data(test_data_, label_id, "test")

        self.dev_data = self.test_data

    def init_workflow(self):
        pass

    def fit(self):
        if not os.path.exists(self.model_save_path):
            os.makedirs(self.model_save_path, exist_ok=True)

        self.config.model_save_path = self.model_save_path

        with codecs.open(os.path.join(self.model_save_path, 'params_config.json'), 'w', encoding='utf-8') as fd:
            json.dump(vars(self.config), fd, indent=4, ensure_ascii=False)

        self.trainer = Trainer(self.config,
                               train_dataset=self.train_data,
                               dev_dataset=self.dev_data,
                               test_dataset=self.test_data)
        self.trainer.train()

    def eval(self):
        self.trainer.load_model()
        eval_results = self.trainer.evaluate1("test")
        print(eval_results)

if __name__ == '__main__':
    config_params = {
        "algorithm_id": 19,
        "hyper_param_strategy": "CUSTOMED",
        "hyper_param": {
            "ADDITIONAL_SPECIAL_TOKENS": [],
            "model_dir": "./model_all",
            "data_dir": "./data",
            "model_type": "bert",
            "seed": 1234,
            "train_batch_size": 32,
            "eval_batch_size": 32,
            "max_seq_len": 195,
            "max_seq_len2": 256,
            "learning_rate": 5e-5,
            "num_train_epochs": 10,  # 5， 20
            "weight_decay": 0.0,
            "gradient_accumulation_steps": 1,
            "adam_epsilon": 1e-8,
            "max_grad_norm": 1.0,
            "max_steps": -1,
            "warmup_steps": 0,
            "dropout_rate": 0.1,
            "logging_steps": 1000,
            "save_steps": 1000,
            "neg_num": 5,
            "no_cuda": False,
            "ignore_index": 0,
            "do_train": True,
            "do_eval": True,
            "sentence_num": 2,
            "is_heading": True,
            "ntn": False
        },
        "train_file_url": [],
        "job_name": "bert_classification_0"
    }

    config_params["hyper_param"]["model_dir"] = "./model_{}".format("all")
    bt = BertClassificationTrainModelHandler(config_params)
    bt.data_process()
    bt.fit()
    bt.eval()

Sichuan Innovation Digital Contest/src/bert_classification_train.py


import re
import time

# from selenium import webdriver
from selenium import webdriver
from bs4 import BeautifulSoup
from lmf.dbv2 import db_write
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.by import By
from selenium.common.exceptions import NoSuchElementException,StaleElementReferenceException
from selenium.common.exceptions import WebDriverException
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC

#初始化
driver=webdriver.Chrome()
# driver.maximize_window()

# driver.set_page_load_timeout(3)
# try:
driver.get('http://www.ccggzy.gov.cn/qxxxgk/003002/003002001/CountyPurhcaseNotice.html')
# except:
    # driver.execute_script('javascript:window.stop()')

#第一个等待
locator=(By.XPATH,'//tbody[@id="showList"]/tr[1]/td[2]/a')
WebDriverWait(driver,10).until(EC.presence_of_element_located(locator))
url=driver.current_url
PAGE=[]
CC_TEXT=[]
#
# for i in range(1,17):
#     if i != 1:
#         val = driver.find_element_by_xpath('//tbody[@id="showList"]/tr[1]/td[2]/a').text
#         driver.find_element_by_xpath('(//div[@class="ewb-screen-list"])[1]/span[{}]'.format(i)).click()
#         #
#         locator = (By.XPATH, '//tbody[@id="showList"]/tr[1]/td[2]/a[not(contains(string(),"%s"))]'%val)
#         WebDriverWait(driver, 10).until(EC.presence_of_element_located(locator))
#         locator = (By.XPATH, '//span[@class="pg_maxpagenum"]')
#         WebDriverWait(driver, 10).until(EC.presence_of_element_located(locator))
#         time.sleep(0.5)
#     page = driver.find_element_by_xpath('//span[@class="pg_maxpagenum"]').text.strip()
#     cc_text=driver.find_element_by_xpath('//span[@class="ewb-screen-name current"]').text
#
#     print(page)
#     total_ = int(re.findall('/(\d+)', page)[0])
#     PAGE.append(total_)
#     CC_TEXT.append(cc_text)
# print(PAGE)
# print(CC_TEXT)
# print(sum(PAGE))
#
#
# #获取总页数
# page=driver.find_element_by_xpath('//span[@class="pg_maxpagenum"]').text
# print(page)
# total=re.findall('/(\d+)',page)[0]
# total=int(total)
# print(total)

data=[]

html = driver.page_source
soup = BeautifulSoup(html, 'lxml')
div=soup.find('tbody',id='showList')
trs=div.find_all('tr')

for tr in trs:

    tds=tr.find_all('td')
    href=tds[1].a['href']
    name=tds[1].a.get_text().strip()

    if name.startswith('【'):
        gg_type=re.findall(r'^【(.+?)】',name)[0]
        name=name.split('】',maxsplit=1)[1]
    else:
        gg_type=None

    ggstart_time=tds[2].get_text()

    if 'http' in href:
        href=href
    else:
        href='http://www.ccggzy.gov.cn'+href

    tmp = [gg_type,name, ggstart_time, href]

    print(tmp)

driver.quit()


#!/usr/bin/env python2
# vim:fileencoding=utf-8

"""
Script to download R2 shows. See `r2get -h` for usage.

Tech notes
----------

Passed parameters:

$SHOW - show name (case-insensitive)
$DATE - show date (optional)

Operation:

1. If $DATE not set, get latest show date from [1].
2. Get show $ID and cover art URL from [2].
3. Fetch show page from [3].
4. Extract RTMP stream URLs.
5. Fetch streams and cover art.
6. Convert streamdumps to a suitable container via ffmpeg.
7. Apply tags, including cover art.

References:

API = "http://r2.err.ee/api/loader"

[1] $API/airtimesforprogram/$SHOW?channel=raadio2&ShowAll=False
[2] $API/GetTimeLineDay/?$DATE
[3] http://r2.err.ee/v/$SHOW/saated/$ID
"""

from __future__ import print_function

import argparse
import base64
import calendar
import datetime
import dateutil.parser
import mutagen.id3
import mutagen.mp4
import mutagen.easymp4
import mutagen.oggvorbis
import mutagen.flac
import json
import os
import re
import stat
import subprocess
import tempfile
import sys
import urllib
import urlparse

from bs4 import BeautifulSoup

AIRTIMES_API_URL = "http://r2.err.ee/api/loader/airtimesforprogram/"
SHOW_API_URL = "http://otse.err.ee/api/schedule/"
IMG_ROOT_URL = "http://static.err.ee/gridfs"

LOGLEVEL_DEFAULT = 0
LOGLEVEL_DEBUG = 1
LOGLEVEL_INFO = 2

# API access routines

def get_last_showdate(show_id):

	# Return last airdate of a show.

	showid = re.sub(r"\s+", "", show_id)
	url = \
		AIRTIMES_API_URL + \
		showid + "?channel=raadio2&ShowAll=False"

	debug("Loading air times from " + url, LOGLEVEL_DEBUG)
	f = urllib.urlopen(url)
	tree = json.load(f)

	if ((type(tree) is not list) or len(tree) == 0):
		err("Did not retrieve any show times for \"" + show_id + "\"")
	return parse_date(tree[0]["AirDate"])

def get_show_attrs(show_id, date, partial_name):

	# Return the canonical name, GUID and artwork URL
	# of a show on a given date.

	url = \
		SHOW_API_URL + "GetTimelineDay?channel=r2" + \
		"&year=" + str(date.year) + "&month=" + str(date.month) + "&day=" + str(date.day)

	debug("Fetching show attributes from " + url, LOGLEVEL_DEBUG)

	f = urllib.urlopen(url)
	tree = json.load(f)
	lname = show_id.lower()
	arturl = None
	artopts = None

	def is_name_match(header_entry):
		header_entry = header_entry.lower()

		if (partial_name):
			return (True if header_entry.find(lname) != -1 else False)
		else:
			return header_entry == lname

	for i in tree:
		header_entry = i["Header"];
		debug("\tHeader entry: " + header_entry, LOGLEVEL_INFO)

		if (is_name_match(header_entry)):
			imgurl = None
			pageurl = None

			if (i["Image"]):
				imgurl = ""
				imgurl += IMG_ROOT_URL + "/" + i["Image"]
				if (i["ImageResizerOptions"]):
					imgurl += i["ImageResizerOptions"]
				else:
					imgurl += "?width=1200"

			if (i["Url"]):
				pageurl = i["Url"]
			else:
				raise KeyError("Cannot get attribute \"Url\" for \"" + show_id + "\"")

			return (
				i["Header"],
				pageurl,
				imgurl
			)

	raise KeyError(
		"Cannot get show attributes for \"" + show_id +
		"\" on " + str(date)
	)

def get_show_streams(url):

	# Return the RTMP stream URLs of a show with a given GUID. Attempts to
	# scrape for "RR.*m4a" streams first; if none found, resorts to any
	# *.m4a streams.

	streams = []
	f = urllib.urlopen(url)
	soup = BeautifulSoup(f, "html.parser")

	# Helper routine for tight / loose condition

	def find_streams(soup, tightmatch):
		found_streams = []
		tightmatch = r".*?RR" if tightmatch else ""

		for stag in soup.findAll("script"):
			suris = re.findall(r"media\.err\.ee" + tightmatch + ".*?m4a", str(stag))

			for suri in (suris):
				suri = "rtmp://" + re.sub("r2/@", "r2/", suri);
				found_streams.append(suri)

		return found_streams

	# Scrape the DOM

	debug("Scraping for show streams from " + url, LOGLEVEL_DEBUG)
	streams = find_streams(soup, True)

	if (not len(streams)):
		debug("Cannot find any RR* streams, attempting to scrape any streams", LOGLEVEL_DEBUG)
		streams = find_streams(soup, False)

	streams = list_uniq(streams)

	if (len(streams) < 1):
		raise RuntimeError("Cannot retrieve any stream URLs from " + url)
	return streams

# Audio retrieval and packaging routines

def download_audio(streams, retries, quick_test=False):

	# Dump raw RTMP streams into temporary files.

	dumps = []

	for s in streams:
		d = tempfile.NamedTemporaryFile(suffix='.m4a')
		args = ["rtmpdump", "-v", "-#", "-o", d.name, "-r", s]
		resume_flag = False

		debug("Dumping " + s + " to " + d.name)

		if (quick_test):
			args += ["-B", "30"]  # call instead check_call because
			subprocess.call(args) # rtmpdump returns non-zero w/ -B
		else:
			for i in range(0, retries):
				if (i > 0):
					debug("Retrying (" + str(i+1) + " / " + str(retries) + ")")
					if (not resume_flag):
						args += ["-e"]
						resume_flag = True

				try:
					subprocess.check_call(args)
				except subprocess.CalledProcessError as e:
					if (e.returncode == 2 and i < (retries-1)): # incomplete transfer
						continue
					else:
						err("Failed fetching stream after " + str(retries) + " times", e)
				break

		dumps.append(d)

	return dumps

def package_audio(show_id, showname, showdate, show_imgurl, infns, outputs):

	# Package dumped raw RTMP stream into a proper m4a (aka ipod)
	# container, apply tags including cover art. infns can be a list of
	# strings (file names) or a list of NamedTemporaryFiles. outputs is
	# a dict of output ID - directory mappings.

	showfn_prefix = re.sub(r"\s+", "-", show_id.lower())

	# Fetch cover image

	coverf = urllib.urlopen(show_imgurl)
	if (coverf.info().gettype() != "image/jpeg" or coverf.getcode() != 200):
		warn("Cannot retrieve JPEG cover art")
	cover = coverf.read()

	# Loop over input stream dumps

	for i, infn in enumerate(infns):
		if (len(infns) <= 1):
			showfn_suffix = "-1"
		else:
			showfn_suffix = "-" + str(i + 1)
		if (type(infn) is not str):
			infn = infn.name

		show_index = re.sub(r"-", " ", showfn_suffix)
		show_title = showname + " " + str(showdate) + show_index

		# Loop over output formats

		for k in outputs:
			outdir = outputs[k]
			showfn = outdir + "/" + showfn_prefix + showfn_suffix

			if (k == "mp3"):
				showfn += ".mp3"
				package_mp3(infn, showfn, show_title, cover)
			if (k == "ogg"):
				showfn += ".ogg"
				package_ogg(infn, showfn, show_title, cover)
			elif (k == "mp4"):
				showfn += ".m4a"
				package_mp4(infn, showfn, show_title, cover)

def package_mp4(infn, outfn, title, cover):
	subprocess.check_call(
		["ffmpeg", "-y", "-i", infn, "-acodec", "copy", outfn])

	tags = mutagen.mp4.MP4(outfn)
	cover = [mutagen.mp4.MP4Cover(cover)]

	tags["\xa9ART"] = "R2"
	tags["\xa9nam"] = title
	tags["\xa9alb"] = "R2"
	tags["covr"] = cover
	tags.save()

def package_mp3(infn, outfn, title, cover):
	subprocess.check_call(
		["ffmpeg", "-y", "-i", infn, "-acodec", "libmp3lame", "-q:a", "1", outfn])

	tags = mutagen.id3.ID3(outfn)
	tags.add(mutagen.id3.TPE1(encoding=3, text="R2"))
	tags.add(mutagen.id3.TIT2(encoding=3, text=title))
	tags.add(mutagen.id3.TALB(encoding=3, text="R2"))
	tags.add(mutagen.id3.APIC(encoding=3, type=3, data=cover))
	tags.save()

def package_ogg(infn, outfn, title, cover):
	subprocess.check_call(
		["ffmpeg", "-y", "-i", infn, "-codec:a", "libvorbis", "-qscale:a", "6", outfn])

	ogg = mutagen.oggvorbis.OggVorbis(outfn)
	ogg.tags["artist"] = u"R2"
	ogg.tags["title"] = title
	ogg.tags["album"] = u"R2"
	write_ogg_image(ogg, cover)
	ogg.save()

# Auxiliary routines

def parse_date(s):

	# Coerce a string into a date object.

	d = dateutil.parser.parse(s)
	return datetime.date(d.year, d.month, d.day)

def parse_command_line():
	parser = argparse.ArgumentParser(description = "Fetch a show from R2.")
	parser.add_argument(
		"-d", "--date",
		help="specify desired show airdate, e.g. \"15\", \"2015-02-12\""
	)
	parser.add_argument(
		"-s", "--skip",
		help="skip download, use specified filename as audio input"
	)
	parser.add_argument(
		"-q", "--quick-test",
		action="store_true",
		help="quick sanity test of the pipeline, download 30 first seconds only"
	)
	parser.add_argument(
		"-p", "--partial-name",
		action="store_true",
		help="allow partial match for show name; only makes sense with -d"
	)
	parser.add_argument(
		"-v", "--verbose",
		action="count",
		help="increase verbosity"
	)
	parser.add_argument(
		"-r", "--retry", type=int, default=5,
		help="number of retries in case of incomplete transfers, default 5"
	)
	parser.add_argument(
		"-D", "--mp4-dir", default=".",
		help="mp4 (unmodified stream) output directory"
	)
	parser.add_argument(
		"--mp3-dir", default=None,
		help="mp3 output directory"
	)
	parser.add_argument(
		"--ogg-dir", default=None,
		help="ogg output directory"
	)
	parser.add_argument("show_id")
	return parser.parse_args()

def gather_outputs(args):
	o = dict()

	if (args.mp4_dir):
		o["mp4"] = args.mp4_dir
	if (args.mp3_dir):
		o["mp3"] = args.mp3_dir
	if (args.ogg_dir):
		o["ogg"] = args.ogg_dir

	for k in o:
		if (not os.access(o[k], os.F_OK or os.W_OK)):
			err(o[k] + " does not exist or isn't writable")

	return o

def write_ogg_image(ogg, cover):

	# Write an image block to OGG Vorbis, see
	# http://mutagen.readthedocs.io/en/latest/user/vcomment.html

	picture = mutagen.flac.Picture()
	picture.data = cover
	picture.type = 3
	picture.mime = u"image/jpeg"

	base64_picture = base64.b64encode(picture.write())
	ogg["metadata_block_picture"] = [base64_picture.decode("ascii")]

def debug(msg, verbosity=0):
	if (verbosity > (debug.verbosity or 0)):
		return
	try:
		print(msg, file=sys.stderr)
	except UnicodeEncodeError as e:
		err(
			"Please set PYTHONIOENCODING=UTF-8 to avoid encoding errors " +
			"in case of badly detected or non-Unicode stderr", e
		)

def warn(msg, is_error=0):
	if (is_error):
		msg = "ERROR: " + msg
	else:
		msg = "WARNING: " + msg
	debug(msg)

def err(msg, e=None):

	# If exception `e` is passed, raise it instead of sys.exit-ing

	warn(msg, is_error=1)

	if (e == None):
		sys.exit(1)
	else:
		raise e

def debug_set_verbosity(v):
	debug.verbosity = v;

def list_uniq(l):
	s = set()
	r = []

	for i in (l):
		if (i not in s):
			r.append(i)
			s.add(i)

	return r

def main():

	# show_id is the identifier from the command line, possibly
	# partial, used for file names. show_name is the name from R2.

	args = parse_command_line()
	show_id = args.show_id.strip()
	streams = None
	infns = None
	outputs = None

	debug_set_verbosity(args.verbose)
	outputs = gather_outputs(args)

	if (args.skip):
		infns = [args.skip]
	if (args.date):
		showdate = parse_date(args.date)
	else:
		showdate = get_last_showdate(show_id)

	(showname, show_url, show_imgurl) = \
		get_show_attrs(show_id, showdate, args.partial_name)
	debug("Show name:\t" + showname)
	debug("Show date:\t" + str(showdate))
	debug("Image URL:\t" + show_imgurl)

	if (not infns):
		streams = get_show_streams(show_url)
		debug("Stream URLs:\t" + str(streams))
		infns = download_audio(streams, args.retry, args.quick_test)
	package_audio(show_id, showname, showdate, show_imgurl, infns, outputs)

main()


import os
import re
import json
import shutil

from .bert.modeling import BertConfig

from . import data_preprocessing
from .utils import create_path, EOS_TOKEN, get_or_make_label_encoder

class Params():
    def __init__(self):

        self.run_problem_list = []

        self.problem_type = {
            'WeiboNER': 'seq_tag',
            'WeiboFakeCLS': 'cls',
            'WeiboSegment': 'seq_tag',
            'WeiboPretrain': 'pretrain',
            'CWS': 'seq_tag',
            'NER': 'seq_tag',
            'CTBPOS': 'seq_tag',
            'CTBCWS': 'seq_tag',
            'ascws': 'seq_tag',
            'msrcws': 'seq_tag',
            'pkucws': 'seq_tag',
            'cityucws': 'seq_tag',
            'bosonner': 'seq_tag',
            'msraner': 'seq_tag',
            'POS': 'seq_tag',
            'weibo_fake_seq2seq_tag': 'seq2seq_tag',
            'weibo_fake_seq_tag': 'seq_tag',
            'ontonotes_ner': 'seq_tag',
            'ontonotes_cws': 'seq_tag',
            'ontonotes_chunk': 'seq2seq_tag',
            'boson_domain': 'cls',
            'Weibo_domain': 'cls',
            'msra_domain': 'cls',
            'as_domain': 'cls',
            'msr_domain': 'cls',
            'pku_domain': 'cls',
            'cityu_domain': 'cls',
            'emotion_analysis': 'cls'
        }
        # self.problem = 'cls'

        self.num_classes = {
            # num of classes of problems
            # including padding if padding is needed
            'WeiboNER': 10,
            'WeiboFakeCLS': 2,
            'WeiboSegment': 4,
            'next_sentence': 2,
            'CWS': 5,
            'NER': 10,
            'CTBPOS': 62,
            'CTBCWS': 5,
            'ascws': 5,
            'msrcws': 5,
            'pkucws': 5,
            'cityucws': 5,
            # 'bosonner': 10,
            # 'msraner': 10,
            'msraner': 8,
            'bosonner': 14,
            'POS': 62,
            'weibo_fake_seq2seq_tag': 4,
            'weibo_fake_seq_tag': 10,
            'ontonotes_ner': 18,
            'ontonotes_cws': 5,
            # add '[SEP]' as eos token
            'ontonotes_chunk': 194,
            'boson_domain': 3,
            'Weibo_domain': 3,
            'msra_domain': 3,
            'as_domain': 4,
            'msr_domain': 4,
            'pku_domain': 4,
            'cityu_domain': 4,
            'emotion_analysis': 2
        }

        self.data_num_dict = {
            'CWS': 157952,
            'NER': 60000,
            'CTBPOS': 47400,
            'CTBCWS': 47400,
            'POS': 47400,
            'ascws': 708953,
            'msrcws': 86924,
            'cityucws': 53019,
            'pkucws': 19056,
            'msraner': 46364,
            'bosonner': 30000,
            'ontonotes_ner': 39086,
            'ontonotes_cws': 39086,
            'ontonotes_chunk': 39086,
            'boson_domain': 10000,
            'msra_domain': 46364,
            'as_domain': 708953,
            'msr_domain': 86924,
            'pku_domain': 53019,
            'cityu_domain': 19056,
            'emotion_analysis': 130000
        }

        # specify this will make key reuse values top
        # that it, WeiboNER problem will use NER's top
        self.share_top = {
            'CTBCWS': 'CWS',
            'ascws': 'CWS',
            'msrcws': 'CWS',
            'pkucws': 'CWS',
            'cityucws': 'CWS',
            'CTBPOS': 'POS',
            'boson_domain': 'ner_domain',
            'Weibo_domain': 'ner_domain',
            'msra_domain': 'ner_domain',
            'as_domain': 'cws_domain',
            'msr_domain': 'cws_domain',
            'pku_domain': 'cws_domain',
            'cityu_domain': 'cws_domain'
        }

        self.eos_id = {
        }

        self.multitask_balance_type = 'data_balanced'
        # self.multitask_balance_type = 'problem_balanced'

        # logging control
        self.log_every_n_steps = 100
        self.detail_log = True

        # training
        self.init_lr = 2e-5
        self.batch_size = 32
        self.train_epoch = 15
        self.freeze_step = 0
        self.prefetch = 5000

        # hparm
        self.dropout_keep_prob = 0.9
        self.max_seq_len = 128
        self.use_one_hot_embeddings = True
        self.label_smoothing = 0.1
        self.crf = True

        # seq2seq
        self.decoder_num_hidden_layers = 3
        self.beam_size = 10
        self.init_decoder_from_encoder = False
        self.beam_search_alpha = 0.6
        self.decode_max_seq_len = 90

        # multitask training
        self.label_transfer = False
        self.augument_mask_lm = False
        self.augument_rate = 0.5
        self.distillation = False
        self.mutual_prediction = False

        # random replace punctuation by some prob to
        # ease the punctuation sensitive problem
        self.punc_replace_prob = 0.0
        self.punc_list = list(',.!?！。？，、')
        self.hidden_gru = False
        self.label_transfer_gru = False
        # if None, we will use the same hidden_size as inputs
        # e.g. # of labels
        self.label_transfer_gru_hidden_size = None

        # bert config
        self.init_checkpoint = 'chinese_L-12_H-768_A-12'

        # pretrain hparm
        self.dupe_factor = 10
        self.short_seq_prob = 0.1
        self.masked_lm_prob = 0.15
        self.max_predictions_per_seq = 20
        self.mask_lm_hidden_size = 768
        self.mask_lm_hidden_act = 'gelu'
        self.mask_lm_initializer_range = 0.02

        self.label_transfer_problem = None

        # get generator function for each problem
        self.read_data_fn = {}
        for problem in self.problem_type:
            try:
                self.read_data_fn[problem] = getattr(
                    data_preprocessing, problem)
            except AttributeError:
                raise AttributeError(
                    '%s function not implemented in data_preprocessing.py' % problem)

    def assign_problem(self, flag_string: str, gpu=2, base_dir=None, dir_name=None):
        """Assign the actual run problem to param. This function will
        do the following things:

        1. parse the flag string to form the run_problem_list
        2. create checkpoint saving path
        3. calculate total number of training data and training steps
        4. scale learning rate with the number of gpu linearly

        Arguments:
            flag_string {str} -- run problem string
            example: CWS|POS|WeiboNER&WeiboSegment

        Keyword Arguments:
            gpu {int} -- number of gpu use for training, this
                will affect the training steps and learning rate (default: {2})
            base_dir {str} -- base dir for ckpt, if None,
                then "tmp" is assigned (default: {None})
            dir_name {str} -- dir name for ckpt, if None,
                will be created automatically (default: {None})
        """
        self.problem_str = flag_string
        # Parse problem string
        self.run_problem_list = []
        for flag_chunk in flag_string.split('|'):

            if '&' not in flag_chunk:
                problem_type = {}
                problem_type[flag_chunk] = self.problem_type[flag_chunk]
                self.run_problem_list.append(problem_type)
            else:
                problem_type = {}
                for problem in flag_chunk.split('&'):
                    problem_type[problem] = self.problem_type[problem]
                self.run_problem_list.append(problem_type)
        if (self.label_transfer or self.mutual_prediction) and self.label_transfer_problem is None:
            self.label_transfer_problem = [p for p in self.run_problem_list]

        problem_list = sorted(re.split(r'[&|]', flag_string))

        # create dir and get vocab, config
        base = base_dir if base_dir is not None else 'tmp'

        dir_name = dir_name if dir_name is not None else '_'.join(
            problem_list)+'_ckpt'
        self.ckpt_dir = os.path.join(base, dir_name)
        create_path(self.ckpt_dir)
        self.params_path = os.path.join(self.ckpt_dir, 'params.json')
        try:
            shutil.copy2(os.path.join(self.init_checkpoint,
                         'vocab.txt'), self.ckpt_dir)
            shutil.copy2(os.path.join(self.init_checkpoint,
                                    'bert_config.json'), self.ckpt_dir)
        except FileNotFoundError:
            pass
        self.vocab_file = os.path.join(self.ckpt_dir, 'vocab.txt')
        self.bert_config = BertConfig.from_json_file(
            os.path.join(self.ckpt_dir, 'bert_config.json'))
        self.bert_config_dict = self.bert_config.__dict__
        with open(self.vocab_file, 'r', encoding='utf8') as vf:
            self.vocab_size = len(vf.readlines())

        # update data_num and train_steps
        self.data_num = 0
        for problem in problem_list:
            if problem not in self.data_num_dict:
                self.data_num += len(
                    list(self.read_data_fn[problem](self, 'train')))
                self.data_num_dict[problem] = len(
                    list(self.read_data_fn[problem](self, 'train')))
            else:
                self.data_num += self.data_num_dict[problem]

        self.shuffle_buffer = min([200000, self.data_num])

        if self.problem_type[problem] == 'pretrain':
            dup_fac = self.dupe_factor
        else:
            dup_fac = 1
        self.train_steps = int((
            self.data_num * self.train_epoch * dup_fac) / (self.batch_size*gpu))
        self.num_warmup_steps = int(0.1 * self.train_steps)

        # linear scale learing rate
        self.lr = self.init_lr * gpu

        # get eos_id
        for problem in problem_list:
            try:
                le = get_or_make_label_encoder(self, problem, 'predict')
                if EOS_TOKEN in le.encode_dict:
                    self.eos_id[problem] = le.transform([EOS_TOKEN])[0]
            except FileNotFoundError:
                pass

        self.to_json()

    @property
    def features_to_dump(self):
        # training
        return [
                'multitask_balance_type',
                'init_lr',
                'batch_size',
                'train_epoch',
                'freeze_step',
                'augument_mask_lm',
                'augument_rate',
                'label_transfer',

                # hparm
                'dropout_keep_prob',
                'max_seq_len',
                'use_one_hot_embeddings',
                'label_smoothing',
                'crf',

                'decoder_num_hidden_layers',
                'beam_size',
                'init_decoder_from_encoder',
                'beam_search_alpha',
                'decode_max_seq_len',
                'label_transfer',
                'augument_mask_lm',
                'augument_rate',
                'distillation',

                # pretrain hparm
                'dupe_factor',
                'short_seq_prob',
                'masked_lm_prob',
                'max_predictions_per_seq',
                'mask_lm_hidden_size',
                'mask_lm_hidden_act',
                'mask_lm_initializer_range',
                'punc_replace_prob',
                'hidden_gru',
                'label_transfer_gru',
                'label_transfer_gru_hidden_size',
                'mutual_prediction']

    def to_json(self):
        dump_dict = {}
        for att in self.features_to_dump:
            value = getattr(self, att)
            dump_dict[att] = value

        with open(self.params_path, 'w', encoding='utf8') as f:
            json.dump(dump_dict, f)

    def from_json(self, json_path=None):
        params_path = json_path if json_path is not None else self.params_path
        with open(params_path, 'r', encoding='utf8') as f:
            dump_dict = json.load(f)
        for att in dump_dict:
            setattr(self, att, dump_dict[att])


import sys

from PyQt5 import QtCore, QtGui, QtWidgets
from PyQt5.QtWidgets import QOpenGLWidget, QWidget, QApplication, QVBoxLayout, QHBoxLayout, QPushButton
from PyQt5.QtWidgets import QMainWindow, QColorDialog
from PyQt5.QtGui import QPainter, QColor, QFont, QPixmap, QIcon, QImage, QClipboard
from PyQt5.QtCore import Qt, QPoint, QRect

class MainWindowLogic(QMainWindow):
    # Текущий цвет, которым будут рисоваться новые элементы
    draw_color_now = QColor(0, 0, 0)

    # Набор доступных инструментов
    draw_tools = {}

    # Текущий инструмент
    draw_tool_now_id = "draw_rectangle"

    def __init__(self, ui_init):
        super().__init__()
        self.ui = ui_init
        self.ui.setupUi(self)

        self.is_drawing = True

        # Создаем изображение, на котором будет рисовать пользователь
        self.drawing_surface = QImage(QPixmap(self.rect().size()))
        self.drawing_surface.fill(Qt.white)
        # Создаем буферное изображение для инструментов, оно будет отрисовываться поверх основного
        self.tool_surface = QPixmap(self.rect().size())#QImage(QPixmap(self.rect().size()))
        self.tool_surface.fill(QColor(0, 0, 0, 0))

        #Различная информация для инструментов
        self.tool_data = {}
        self.tool_data.setdefault("color",QColor(0,0,0))

        # Назначаем обработчики событий
        self.add_functions()

        # Инициализируем инструменты
        self.draw_tools.setdefault("draw_rectangle",ToolController_Figure_Rectangle(self))
        self.draw_tools.setdefault("draw_ellipse",ToolController_Figure_Ellipse(self))
        self.draw_tools.setdefault("draw_line",ToolController_Figure_Line(self))
        self.draw_tools.setdefault("draw_point",ToolController_Figure_Point(self))
        self.draw_tools.setdefault("draw_fill",ToolController_Fill(self))
        self.draw_tools.setdefault("select",ToolController_Select(self))
        self.draw_tool_now = self.draw_tools["draw_rectangle"]

        # Выводим окно
        self.show()

    def add_functions(self):

        self.ui.actionRectangle.triggered.connect(lambda: self.changeTool("draw_rectangle"))
        self.ui.actionChangeColor.triggered.connect(lambda: self.changeColor())
        self.ui.actionEllipse.triggered.connect(lambda: self.changeTool("draw_ellipse"))
        self.ui.actionLine.triggered.connect(lambda: self.changeTool("draw_line"))
        self.ui.actionPoint.triggered.connect(lambda: self.changeTool("draw_point"))
        self.ui.actionFill.triggered.connect(lambda: self.changeTool("draw_fill"))
        self.ui.actionSelect.triggered.connect(lambda: self.changeTool("select"))

    def changeTool(self,new_tool_id):
        self.draw_tool_now_id = new_tool_id
        self.draw_tool_now = self.draw_tools[new_tool_id]

    def changeColor(self):
        self.tool_data["color"] = QColorDialog.getColor()

        pixmap = QPixmap(":/tool_bar/tool_collor.png")
        mask = pixmap.createMaskFromColor(QColor(0,0,0),Qt.MaskOutColor)
        pixmap.fill(self.tool_data["color"])
        pixmap.setMask(mask)

        self.ui.actionChangeColor.setIcon(QIcon(pixmap))

    def paintEvent(self, event):
        painter = QPainter(self)
        painter.begin(self)
        painter.setPen(self.draw_color_now)
        painter.drawImage(QPoint(), self.drawing_surface)
        painter.drawPixmap(QPoint(), self.tool_surface)

        painter.end()

    def mousePressEvent(self, event):
        self.draw_tool_now.mouse_press_handler(event)

    def mouseMoveEvent(self, event):
        self.draw_tool_now.mouse_move_handler(event)

    def mouseReleaseEvent(self, event):
        self.draw_tool_now.mouse_release_handler(event)

#Базовый класс для инструментов
class ToolController:
    def __init__(self,window):
        self.main_window = window
        self.main_surface = window.drawing_surface
    def mouse_press_handler(self, event):
        pass
    def mouse_move_handler(self, event):
        pass
    def mouse_release_handler(self, event):
        pass

#Базовый класс, отвечающий за рисование фигур
class ToolController_Figure(ToolController):
    is_draw_start = False
    is_accumulation = False

    def draw_figure(self, painter):
        pass

    def mouse_press_handler(self, event):
        self.point0 = event.pos()
        self.point1 = event.pos()
        self.is_draw_start = True

        # Поскольку размер фигуры задается перемещением мыши, нет смысла дублировать код
        # который отвечает за отрисовку фигуры на буферной поверхности
        self.mouse_move_handler(event)

    def mouse_move_handler(self, event):
        if self.is_draw_start:
            self.point1 = event.pos()

            # Сначала временная поверхность для рисования отчищается в цвет с нулевой прозрачностью
            # Затем на ней рисуется, как будет выглядить фигура при текущих параметрах
            painter = QPainter(self.main_window.tool_surface)
            painter.begin(self.main_window.tool_surface)
            painter.setPen(self.main_window.tool_data['color'])

            if (not self.is_accumulation):
                self.main_window.tool_surface.fill(QColor(0, 0, 0, 0))

            self.draw_figure(painter)

            painter.end()
            self.main_window.update()

    def mouse_release_handler(self, event):
        self.is_draw_start = False
        self.point1 = event.pos()

        painter = QPainter(self.main_window.drawing_surface)
        #painter.drawImage(QPoint(), self.main_window.drawing_surface)
        painter.setPen(self.main_window.tool_data['color'])
        painter.drawPixmap(QPoint(), self.main_window.tool_surface)

        self.main_window.update()

#Инструмент, отвечающий за рисование прямоугольника
class ToolController_Figure_Rectangle(ToolController_Figure):
    def draw_figure(self, painter):
        rect = QRect(self.point0, self.point1)
        painter.drawRect(rect.normalized())

#Инструмент, отвечающий за рисование эллипса
class ToolController_Figure_Ellipse(ToolController_Figure):
    def draw_figure(self, painter):
        rect = QRect(self.point0, self.point1)
        painter.drawEllipse(rect)

#Инструмент, отвечающий за рисование прямой
class ToolController_Figure_Line(ToolController_Figure):
    def draw_figure(self, painter):
        painter.drawLine(self.point0, self.point1)

#Инструмент, отвечающий за рисование точки
class ToolController_Figure_Point(ToolController_Figure):
    def __init__(self,window):
        super(ToolController_Figure_Point, self).__init__(window)
        self.is_accumulation = True

    def draw_figure(self, painter):
        painter.drawLine(self.point0, self.point1)

    def mouse_move_handler(self,event):
        if self.is_draw_start:
            self.point0 = self.point1
        super(ToolController_Figure_Point, self).mouse_move_handler(event)

class ToolController_Fill(ToolController):

    def mouse_press_handler(self, event):

        # Сначала временная поверхность для рисования отчищается в цвет с нулевой прозрачностью
        # Затем на ней рисуется исходное изображение, но с заливкой, как будет выглядить фигура при текущих параметрах
        painter = QPainter(self.main_window.tool_surface)
        painter.begin(self.main_window.tool_surface)
        painter.setPen(self.main_window.tool_data['color'])

        self.main_window.tool_surface.fill(QColor(0, 0, 0, 0))

        # Сама заливка
        image = self.main_window.drawing_surface
        set_checked = set()
        set_for_check = set()
        set_for_check.add((event.pos().x(),event.pos().y()))

        while True:
            self.fill(image, painter, image.pixelColor(event.pos()), set_checked, set_for_check, set_for_check.pop())
            if len(set_for_check)==0:
                break

        painter.end()

        self.main_window.update()

    def mouse_release_handler(self, event):
        painter = QPainter(self.main_window.drawing_surface)
        #painter.drawImage(QPoint(), self.main_window.drawing_surface)
        painter.setPen(self.main_window.tool_data['color'])
        painter.drawPixmap(QPoint(), self.main_window.tool_surface)

        self.main_window.update()

    def fill(self, image, painter, right_color, set_checked,set_for_check,pos):
        set_checked.add(pos)
        painter.drawPoint(int(pos[0]), int(pos[1]))

        for i in (-1,0,1):
            for j in (-1,0,1):
                if not ((pos[0]+i,pos[1]+j) in set_checked) and pos[0]>=0 and pos[1]>=0 and pos[0]<image.width() and pos[1]<image.height() and not (abs(i)==abs(j)):
                    if image.pixelColor(pos[0]+i,pos[1]+j).getRgb() == right_color.getRgb():
                        set_for_check.add((pos[0]+i,pos[1]+j))

class ToolController_Select(ToolController):
    #Процесс выбора. -1=ничего не выбрано,0=выбирается,1=что-то выбрано
    selecting=-1

    #Включен ли режим перемещения
    is_moving=False

    move_start_point=QPoint()
    move_now_point=QPoint()

    def __init__(self,window):
        super(ToolController_Select, self).__init__(window)
        self.selected_image=QImage(QPixmap())

    def mouse_press_handler(self, event):
        if self.selecting==-1:
            self.selecting=0
            self.point0 = event.pos()
            self.point1 = event.pos()
        if self.selecting==1:
            if not QRect(self.point0, self.point1).intersects(QRect(event.pos(), event.pos())):
                self.selecting=-1

                # Сначала временная поверхность для рисования отчищается в цвет с нулевой прозрачностью
                # Затем на ней рисуется, как будет выглядить фигура при текущих параметрах
                painter = QPainter(self.main_window.drawing_surface)
                painter.begin(self.main_window.drawing_surface)

                black_image = QImage(self.selected_image)
                black_image.fill(Qt.black)
                painter.setPen(Qt.white)
                painter.drawImage(self.move_start_point, black_image)
                painter.drawImage(self.move_now_point, self.selected_image)

                painter.end()
                self.main_window.update()
            else:

                self.is_moving=True

    def mouse_move_handler(self, event):
        if self.selecting==0:
            self.point1 = event.pos()

            # Сначала временная поверхность для рисования отчищается в цвет с нулевой прозрачностью
            # Затем на ней рисуется, как будет выглядить фигура при текущих параметрах
            painter = QPainter(self.main_window.tool_surface)
            painter.begin(self.main_window.tool_surface)
            painter.setPen(self.main_window.tool_data['color'])

            self.main_window.tool_surface.fill(QColor(0, 0, 0, 0))

            rect = QRect(self.point0, self.point1)
            painter.drawRect(rect.normalized())

            painter.end()
            self.main_window.update()

        if self.selecting==1 and self.is_moving:
            self.move_now_point=event.pos()

            # Сначала временная поверхность для рисования отчищается в цвет с нулевой прозрачностью
            # Затем на ней рисуется, как будет выглядить фигура при текущих параметрах
            painter = QPainter(self.main_window.tool_surface)
            painter.begin(self.main_window.tool_surface)

            self.main_window.tool_surface.fill(QColor(0, 0, 0, 0))
            black_image = QImage(self.selected_image)
            black_image.fill(Qt.black)
            painter.setPen(Qt.white)
            painter.drawImage(self.move_start_point,black_image)
            painter.drawImage(self.move_now_point,self.selected_image)

            painter.end()
        self.main_window.update()

    def mouse_release_handler(self, event):
        if self.selecting == 0:
            self.selecting=1
            self.move_start_point = self.point0
            self.point1 = event.pos()

            self.selected_image=self.main_window.drawing_surface.copy(QRect(self.point0, self.point1))
            buff_test=QApplication.clipboard()
            #buff_test.\
            buff_test.setImage(self.selected_image)
            # Сначала временная поверхность для рисования отчищается в цвет с нулевой прозрачностью
            # Затем на ней рисуется, как будет выглядить фигура при текущих параметрах
            painter = QPainter(self.main_window.tool_surface)
            painter.begin(self.main_window.tool_surface)
            painter.setPen(self.main_window.tool_data['color'])

            self.main_window.tool_surface.fill(QColor(0, 0, 0, 0))

            rect = QRect(self.point0, self.point1)
            painter.drawRect(rect.normalized())

            painter.end()
            self.main_window.update()
        else:
            if self.selecting == 1 and self.is_moving:
                self.move_now_point = event.pos()
                self.is_moving=False
                self.point0=self.move_now_point
                self.point1=QPoint()
                self.point1.setX(self.point0.x()+self.selected_image.width())
                self.point1.setY(self.point0.y()+self.selected_image.height())

        self.main_window.update()


import sys, os, re
import json

from kafka import KafkaConsumer, TopicPartition
consumer = KafkaConsumer()
consumer.assign([TopicPartition('test', 0)])
consumer.seek_to_beginning()

for message in consumer:
  message_bytes = message.value
  message_string = message_bytes.decode()
  message_object = json.loads(message_string)
  print(message_object)